diff -Naur linux-2.6.9/arch/i386/Kconfig linux-2.6.9-lvlintr/arch/i386/Kconfig --- linux-2.6.9/arch/i386/Kconfig 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/Kconfig 2004-12-27 14:23:45.000000000 +0900 @@ -868,6 +868,102 @@ generate incorrect output with certain kernel constructs when -mregparm=3 is used. +# for prioritized interrupt processing +config PRIORITY_INTR + depends on (EXPERIMENTAL && !SMP) && (PREEMPT) + bool "Level based interruption handling" + default n + ---help--- + This adopt interrupt priorities for interrupt handlers + (top half handlers) + to handle interruption orderly(like Solaris). + Current version of this feature does not support SMP Kernel. + +config PRIORITY_INTR_ASSIGN + string "Level assignments" + depends on PRIORITY_INTR + default "" + ---help--- + Prioritized interruption handling facility makes top halves of + kernel threads run in SCHED_FIFO class(except the highest priority + (512), it's called directly as normal linux kernel, not threaded.). + And priority levels of interrupts are expressed as + thread's priority. + Please set interrupts' priority level as following: + IRQ`:'Prioriy,IRQ`:'Prioriy,IRQ`:'Prioriy,...,IRQ`:'Prioriy + Range of priority=1(least significant)..512(most significant) + e.g. To assign IRQ1 as priority 10 and IRQ3 as 12: + 1:10,3:12 + Valid interrupt priority and type of interrupts are described + as follow: + 512 Quick interrupts + 385 -- 511 Reserved for quick interrupts + 257 -- 384 Priority interrupts(included reserved values.) + 129 -- 256 Normal interrupts or softirq + 100 -- 128 Reserved for normal interrupts or softirq + + We recommend configure levels of normal interrupts + between 129 and 256,but you can configure it between 1 and 256 + as you need. + +config PRIORITY_INTR_DEFAULT_LVL + int "Default Level" + range 129 384 + depends on PRIORITY_INTR + default "129" + ---help--- + This is default priority level for top halves. + +config PRIORITY_INTR_NORMAL + bool "Normal interrupt facility " + depends on PRIORITY_INTR + ---help--- + This facility make use of interrupt threads. + +config PRIORITY_PRIO + bool "Hardware assist interupt masking " + depends on PRIORITY_INTR + ---help--- + This facility make use of interrupt threads. + +config PRIORITY_INTR_SOFTIRQ_LVL + int "Priority level of softirqs" + range 1 256 + depends on (PRIORITY_INTR && PRIORITY_INTR_NORMAL) + default "100" + ---help--- + This is default priority level for softirqs. + In general, it should be set a value lower than top halves, and + it is preferable to set it as higher than user processes which + runs on your system. We recommend configure this value + between 129 and 256 and lower than normal interrupts priority + level, but you can configure this value between 1 and 256 as you + need. + +config PRIORITY_PRIVILEGED_IRQ + bool "Privileged interruption handling facility" + depends on PRIORITY_INTR + default n + ---help--- + Quick interrupt handling facility for embedded systems. + If this item is enabled,accepting some interrupts in current + Linux kernel's critical sections. If you set this option, + the highest priority interrupts are treated as quick interruption. + +#config PRIO_INTR_DIAG +# bool "Diagnostic facilities for PRIO_INTR" +# depends on PRIORITY_INTR +# default n +# ---help--- +# kernel dynamic loadable trace facilities. + +#config PRIO_INTR_DIAG_TRACE_TPRI +# bool "Interrupt trace facility" +# depends on PRIO_INTR_DIAG +# default n +# ---help--- +# Task Priority register tracer + endmenu diff -Naur linux-2.6.9/arch/i386/Kconfig.debug linux-2.6.9-lvlintr/arch/i386/Kconfig.debug --- linux-2.6.9/arch/i386/Kconfig.debug 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/Kconfig.debug 2004-12-27 11:42:03.000000000 +0900 @@ -48,6 +48,7 @@ config 4KSTACKS bool "Use 4Kb for kernel stacks instead of 8Kb" + depends on !PRIORITY_INTR help If you say Y here the kernel will use a 4Kb stacksize for the kernel stack attached to each process/thread. This facilitates diff -Naur linux-2.6.9/arch/i386/kernel/Makefile linux-2.6.9-lvlintr/arch/i386/kernel/Makefile --- linux-2.6.9/arch/i386/kernel/Makefile 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/Makefile 2004-12-27 14:23:45.000000000 +0900 @@ -33,9 +33,13 @@ obj-$(CONFIG_EFI) += efi.o efi_stub.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o + EXTRA_AFLAGS := -traditional obj-$(CONFIG_SCx200) += scx200.o +obj-$(CONFIG_PRIORITY_INTR) += tpriops.o lvlintr386.o +obj-$(CONFIG_PRIORITY_PRIVILEGED_IRQ) += crit_intr.o +obj-$(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) += tpri_trace.o # vsyscall.o contains the vsyscall DSO images as __initdata. # We must build both images before we can assemble it. diff -Naur linux-2.6.9/arch/i386/kernel/apic.c linux-2.6.9-lvlintr/arch/i386/kernel/apic.c --- linux-2.6.9/arch/i386/kernel/apic.c 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/apic.c 2004-12-27 11:42:03.000000000 +0900 @@ -1144,7 +1144,11 @@ */ irq_enter(); smp_local_timer_interrupt(®s); +#if defined(CONFIG_PRIORITY_INTR) + irq_exit_nobh(); +#else irq_exit(); +#endif /* CONFIG_PRIORITY_INTR */ } /* @@ -1165,9 +1169,13 @@ ack_APIC_irq(); /* see sw-dev-man vol 3, chapter 7.4.13.5 */ +#if !defined(CONFIG_PRIORITY_INTR) printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n", smp_processor_id()); irq_exit(); +#else + irq_exit_nobh(); +#endif /* CONFIG_PRIORITY_INTR */ } /* @@ -1198,7 +1206,11 @@ */ printk (KERN_DEBUG "APIC error on CPU%d: %02lx(%02lx)\n", smp_processor_id(), v , v1); +#if defined(CONFIG_PRIORITY_INTR) + irq_exit_nobh(); +#else irq_exit(); +#endif /* CONFIG_PRIORITY_INTR */ } /* diff -Naur linux-2.6.9/arch/i386/kernel/crit_intr.c linux-2.6.9-lvlintr/arch/i386/kernel/crit_intr.c --- linux-2.6.9/arch/i386/kernel/crit_intr.c 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/crit_intr.c 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,160 @@ +/* + * arch/i386/kernel/crit_intr.c + * this is ported from PPC4xx. + * Author: Sun Zhitai(sun@cs.fujitsu.co.jp) + * + * 2004 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#undef CRIT_INTR_DEBUG + +/**********************************************************************/ +/* function: is_privilege */ +/* Input: int irq ... irq number */ +/* Output: None */ +/* Return: 0,No; 0x10000,Yes */ +/* Note: None. */ +/* description: check privilege irq */ +/* author: Sun Zhitai */ +/* Revision: V1.1, ported for i386, by Sun Zhitai at March 26, 2004 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +unsigned long +is_privilege(struct pt_regs regs) +{ + int irq = regs.orig_eax & 0xff; + + if(sl_hmask_irq[irq].soft_level == MAX_INTR_LEVEL) { +#if defined(CRIT_INTR_DEBUG) + __asm__("nop\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + ); +#endif /* CRIT_INTR_DEBUG */ + regs.xes |= PRIVILEGED_IRQ_FLAG; + return 1; + } + return 0; +} +/**********************************************************************/ +/* function: ack_critical_interrupt */ +/* Input: int irq ... irq number to ack */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Send ack to UIC */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Oct 21, 2003 */ +/* V1.1, ported for i386, by Sun Zhitai at March 26, 2004 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static inline void +ack_critical_interrupt(int irq) +{ + irq_desc_t *desc = irq_desc + irq; + unsigned long flags; + + spin_lock_interruptsave(&desc->lock, flags); + desc->handler->ack(irq); + spin_unlock_interruptrestore(&desc->lock, flags); +} + +/**********************************************************************/ +/* function: end_critical_interrupt */ +/* Input: int irq ... irq number to ack */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: end critical interrupt */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Oct 21, 2003 */ +/* V1.1, ported for i386, by Sun Zhitai at March 26, 2004 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static inline void +end_critical_interrupt(int irq) +{ +/* send EOI */ + irq_desc_t *desc = irq_desc + irq; + unsigned long flags; + + spin_lock_interruptsave(&desc->lock, flags); + desc->handler->enable(irq); + spin_unlock_interruptrestore(&desc->lock, flags); +} + +/**********************************************************************/ +/* function: CriticalInterruptException */ +/* Input: struct pt_regs *regs ... Interrupt context */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Invoke critical interrupt handlers */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void +CriticalInterruptException(struct pt_regs regs) +{ + int irq; + int status=0; + struct irqaction *action; + irq_desc_t *desc; + irq = regs.orig_eax & 0xff; +#if defined(CRIT_INTR_DEBUG) + { + unsigned long flags; + + local_save_flags(flags); + printk("Call Privileged irq:%d; flags:0x%lx; XES:0x%lx ",irq,flags,regs.xes); + } +#endif /* CRIT_INTR_DEBUG */ + desc = irq_desc + irq; + /* There is no support on SMP. + */ + ack_critical_interrupt(irq); //Ack and mask + /* Note: IRQ_REPLAY/IRQ_PENDING/IRQ_WAITING is never set on + * sigle processor environments. + */ + action = desc->action; + if (unlikely(!action || !action->handler)) { + /* There is no handler. + * This is spurious interrupt. + */ + /* We mask the interrupt not to raise. */ + ++desc->depth; + desc->status |= IRQ_DISABLED; + desc->handler->disable(irq); + goto out; + } + status |= IRQ_INPROGRESS; /* we are handling it */ + + desc->status = status; + HANDLE_IRQ_EVENT(irq, ®s, action); + out: + desc->status &= ~IRQ_INPROGRESS; + /* + * The handler has to deal with interrupts which got + * disabled while the handler was running. + */ + end_critical_interrupt(irq); +} + diff -Naur linux-2.6.9/arch/i386/kernel/entry.S linux-2.6.9-lvlintr/arch/i386/kernel/entry.S --- linux-2.6.9/arch/i386/kernel/entry.S 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/entry.S 2004-12-27 11:42:03.000000000 +0900 @@ -48,6 +48,17 @@ #include #include #include "irq_vectors.h" + +#if defined(DEBUG_STORE_TPR) +#define DBG_HMSK_LVL call check_remember_hardmask_level; \ + movl ORIG_EAX(%esp),%eax; +#define DBG_HMSK_IRET_CHK call check_hardmask_level; +#define DBG_CHECK_TI call check_thread_info; +#else +#define DBG_HMSK_LVL +#define DBG_HMSK_IRET_CHK +#define DBG_CHECK_TI +#endif /* DEBUG_STORE_TPR */ #define nr_syscalls ((syscall_table_size)/4) @@ -94,7 +105,7 @@ pushl %ebx; \ movl $(__USER_DS), %edx; \ movl %edx, %ds; \ - movl %edx, %es; + movl %edx, %es; #define RESTORE_INT_REGS \ popl %ebx; \ @@ -123,6 +134,7 @@ #define RESTORE_ALL \ + DBG_CHECK_TI \ RESTORE_REGS \ addl $4, %esp; \ 1: iret; \ @@ -139,6 +151,26 @@ .long 1b,2b; \ .previous +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +#define __store_intr_tpr(flg) \ + call remember_hardmask_level; \ + movl ES(%esp), %edx; \ + and $ES_MASK, %edx; \ + or flg, %edx; \ + movl %edx, ES(%esp); \ + movl ORIG_EAX(%esp),%eax; +#define __store_tpr(flg) \ + __store_intr_tpr(flg) \ + DBG_HMSK_LVL +#define __restore_tpr \ + call recover_hardmask_level; \ + DBG_HMSK_IRET_CHK + +#else /* !(defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ)) */ +#define __store_intr_tpr(flg) +#define __store_tpr(flg) +#define __restore_tpr +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ ENTRY(lcall7) @@ -146,6 +178,7 @@ # gates, which has to be cleaned up later.. pushl %eax SAVE_ALL + __store_tpr($SYSCALLS_FLAG) movl %esp, %ebp pushl %ebp pushl $0x7 @@ -168,6 +201,7 @@ # gates, which has to be cleaned up later.. pushl %eax SAVE_ALL + __store_tpr($SYSCALLS_FLAG) movl %esp, %ebp pushl %ebp pushl $0x27 @@ -181,6 +215,15 @@ popl %eax jmp syscall_exit +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +ES_MASK = 0x00ffffff /* TPR+ES(use to clean up xes) */ +TPR_MASK = 0x00ff0000 /* Use to obtain TPR MASK LEVEL */ +SYS_IRQS_FLAG = 0x08000000 /* A system interrupt occurred. */ +CRITICAL_FLAG = 0x10000000 /* Critical interrupt occur */ +INTERRUPT_FLAG = 0x20000000 /* High priority/normal interrupt occur */ +EXCEPTION_FLAG = 0x40000000 /* CPU exception occur */ +SYSCALLS_FLAG = 0x80000000 /* A system call is issued. */ +#endif /* * Return to user mode is not as complex as all this looks, * but we want the default path for a system call return to @@ -193,6 +236,15 @@ ret_from_exception: preempt_stop ret_from_intr: +#ifdef CONFIG_PRIORITY_PRIVILEGED_IRQ + movl ES(%esp), %eax # get interrupt vector + testl $CRITICAL_FLAG, %eax # is the privilege ? + jz not_privilege + + jmp restore_all +not_privilege: +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ + GET_THREAD_INFO(%ebp) movl EFLAGS(%esp), %eax # mix EFLAGS and CS movb CS(%esp), %al @@ -218,6 +270,10 @@ jz restore_all testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ? jz restore_all +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + testl $TPR_MASK,ES(%esp) # mask off (Normal INTR path) ? + jnz restore_all +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ movl $PREEMPT_ACTIVE,TI_preempt_count(%ebp) sti call schedule @@ -272,11 +328,11 @@ sti sysexit - # system call handler stub ENTRY(system_call) pushl %eax # save orig_eax SAVE_ALL + __store_tpr($SYSCALLS_FLAG) GET_THREAD_INFO(%ebp) # system call tracing in operation testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%ebp) @@ -294,6 +350,7 @@ testw $_TIF_ALLWORK_MASK, %cx # current->work jne syscall_exit_work restore_all: + __restore_tpr RESTORE_ALL # perform work that needs to be done immediately before resumption @@ -361,6 +418,7 @@ syscall_fault: pushl %eax # save orig_eax SAVE_ALL + __store_tpr($SYSCALLS_FLAG) GET_THREAD_INFO(%ebp) movl $-EFAULT,EAX(%esp) jmp resume_userspace @@ -393,15 +451,45 @@ ALIGN common_interrupt: SAVE_ALL + __store_intr_tpr($INTERRUPT_FLAG) +#ifdef CONFIG_PRIORITY_PRIVILEGED_IRQ + call is_privilege + cmpl $0, %eax # is the privilege ? ; + je not_privilege_in + call CriticalInterruptException + jmp ret_from_intr + +not_privilege_in: +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ + DBG_HMSK_LVL call do_IRQ jmp ret_from_intr +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + +/* to confirm local timer's vector has been rewritten */ +ENTRY(spurious_timer_interrupt) + pushl $0xef-0x100 /* 0xef - 0x100 (256) */ + SAVE_ALL + __store_tpr($SYS_IRQS_FLAG) + call smp_spurious_timer_interrupt + jmp ret_from_intr; + +#define BUILD_INTERRUPT(name, nr) \ +ENTRY(name) \ + pushl $nr-256; \ + SAVE_ALL \ + __store_tpr($SYS_IRQS_FLAG) \ + call smp_/**/name; \ + jmp ret_from_intr; +#else #define BUILD_INTERRUPT(name, nr) \ ENTRY(name) \ pushl $nr-256; \ SAVE_ALL \ call smp_/**/name; \ jmp ret_from_intr; +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ /* The include is where all of the SMP etc. interrupts come from */ #include "entry_arch.h" @@ -427,12 +515,20 @@ movl ES(%esp), %edi # get the function address movl %eax, ORIG_EAX(%esp) movl %ecx, ES(%esp) +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + movl $(__USER_DS), %edx + movl %edx, %ds + movl %edx, %es +#endif + __store_tpr($EXCEPTION_FLAG) movl %esp, %edx pushl %esi # push the error code pushl %edx # push the pt_regs pointer +#if !( defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ)) movl $(__USER_DS), %edx movl %edx, %ds movl %edx, %es +#endif call *%edi addl $8, %esp jmp ret_from_exception @@ -450,6 +546,7 @@ ENTRY(device_not_available) pushl $-1 # mark this as an int SAVE_ALL + __store_tpr($EXCEPTION_FLAG) movl %cr0, %eax testl $0x4, %eax # EM (math emulation bit) jne device_not_available_emulate @@ -525,11 +622,13 @@ nmi_stack_correct: pushl %eax SAVE_ALL + __store_tpr($EXCEPTION_FLAG) movl %esp, %edx pushl $0 pushl %edx call do_nmi addl $8, %esp + __restore_tpr RESTORE_ALL nmi_stack_fixup: diff -Naur linux-2.6.9/arch/i386/kernel/i8259.c linux-2.6.9-lvlintr/arch/i386/kernel/i8259.c --- linux-2.6.9/arch/i386/kernel/i8259.c 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/i8259.c 2004-12-27 11:42:03.000000000 +0900 @@ -29,6 +29,7 @@ #include + /* * This is the 'legacy' 8259A Programmable Interrupt Controller, * present in the majority of PC/AT boxes. @@ -88,11 +89,24 @@ */ unsigned long io_apic_irqs; +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +#define __i8259_intr_save \ + unsigned long f; \ + __kern_intr_save(f) +#define __i8259_intr_restore \ + __kern_intr_restore(f) +#else /* !(defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ)) */ +#define __i8259_intr_save +#define __i8259_intr_restore +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ + void disable_8259A_irq(unsigned int irq) { unsigned int mask = 1 << irq; unsigned long flags; + __i8259_intr_save; + spin_lock_irqsave(&i8259A_lock, flags); cached_irq_mask |= mask; if (irq & 8) @@ -100,6 +114,8 @@ else outb(cached_master_mask, PIC_MASTER_IMR); spin_unlock_irqrestore(&i8259A_lock, flags); + + __i8259_intr_restore; } void enable_8259A_irq(unsigned int irq) @@ -107,6 +123,8 @@ unsigned int mask = ~(1 << irq); unsigned long flags; + __i8259_intr_save; + spin_lock_irqsave(&i8259A_lock, flags); cached_irq_mask &= mask; if (irq & 8) @@ -114,6 +132,8 @@ else outb(cached_master_mask, PIC_MASTER_IMR); spin_unlock_irqrestore(&i8259A_lock, flags); + + __i8259_intr_restore; } int i8259A_irq_pending(unsigned int irq) @@ -122,6 +142,8 @@ unsigned long flags; int ret; + __i8259_intr_save; + spin_lock_irqsave(&i8259A_lock, flags); if (irq < 8) ret = inb(PIC_MASTER_CMD) & mask; @@ -129,6 +151,7 @@ ret = inb(PIC_SLAVE_CMD) & (mask >> 8); spin_unlock_irqrestore(&i8259A_lock, flags); + __i8259_intr_restore; return ret; } @@ -174,6 +197,8 @@ unsigned int irqmask = 1 << irq; unsigned long flags; + __i8259_intr_save; + spin_lock_irqsave(&i8259A_lock, flags); /* * Lightweight spurious IRQ detection. We do not want @@ -206,6 +231,8 @@ outb(0x60+irq,PIC_MASTER_CMD); /* 'Specific EOI to master */ } spin_unlock_irqrestore(&i8259A_lock, flags); + + __i8259_intr_restore; return; spurious_8259A_irq: @@ -294,6 +321,8 @@ { unsigned long flags; + __i8259_intr_save; + spin_lock_irqsave(&i8259A_lock, flags); outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ @@ -329,6 +358,8 @@ outb(cached_slave_mask, PIC_SLAVE_IMR); /* restore slave IRQ mask */ spin_unlock_irqrestore(&i8259A_lock, flags); + + __i8259_intr_restore; } /* diff -Naur linux-2.6.9/arch/i386/kernel/io_apic.c linux-2.6.9-lvlintr/arch/i386/kernel/io_apic.c --- linux-2.6.9/arch/i386/kernel/io_apic.c 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/io_apic.c 2004-12-28 08:56:53.000000000 +0900 @@ -33,6 +33,10 @@ #include #include +#if defined(CONFIG_PRIORITY_INTR) +#include +#include +#endif /* CONFIG_PRIORITY_INTR */ #include #include #include @@ -80,6 +84,41 @@ #else #define vector_to_irq(vector) (vector) #endif +/* append for FJe Linux at May 21, 2004. Sun Zhitai. */ +#if defined(CONFIG_PRIORITY_INTR) +int reassign_flag; +int resetup_IO_APIC_flag = 0; +int highest_pri_vec=0xff; /* the init value */ +int hmask2smask[PRIO_INTR_MAX_VECTOR_UP]; /* as vectors 0 to 255 */ +int smask2hmask[MAX_INTR_LEVEL+1]; /* To search for hard mask with a soft level 1<->512 easily */ +struct sl_hmask_irq sl_hmask_irq[NR_IRQS]; +extern int irq2lvl[NR_IRQS]; /* the soft levels,0 to 512? cmmted by Sun. */ +static int __init reassign_irq_vector(int irq); +#endif /* CONFIG_PRIORITY_INTR */ + +//#define DEBUG_PRIORITY_VECTOR +#undef DEBUG_PRIORITY_VECTOR +#ifdef DEBUG_PRIORITY_VECTOR +static int __init deb_out(void); +#endif /* DEBUG_PRIORITY_VECTOR */ + +//#define DEBUG_PRIORITY_VECTOR_CT +#undef DEBUG_PRIORITY_VECTOR_CT +#ifdef DEBUG_PRIORITY_VECTOR_CT +void kern_raise_irq(int irq); +void kern_down_irq(int irq); +#endif /* DEBUG_PRIORITY_VECTOR_CT */ + +#if defined(CONFIG_PRIORITY_INTR) +#define __io_apic_intr_save \ + unsigned long f; \ + __kern_intr_save(f) +#define __io_apic_intr_restore \ + __kern_intr_restore(f) +#else /* !CONFIG_PRIORITY_INTR */ +#define __io_apic_intr_save +#define __io_apic_intr_restore +#endif /* defined(CONFIG_PRIORITY_PRIO) */ /* * The common case is 1:1 IRQ<->pin mappings. Sometimes there are @@ -89,8 +128,17 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin) { static int first_free_entry = NR_IRQS; +#if defined(CONFIG_PRIORITY_INTR) + static int call_count=0; +#endif struct irq_pin_list *entry = irq_2_pin + irq; - +#if defined(CONFIG_PRIORITY_INTR) + if (unlikely(!call_count)){ + call_count=1; + first_free_entry = NR_IRQS; + } +#endif + while (entry->next) entry = irq_2_pin + entry->next; @@ -171,18 +219,26 @@ { unsigned long flags; + __io_apic_intr_save; + spin_lock_irqsave(&ioapic_lock, flags); __mask_IO_APIC_irq(irq); spin_unlock_irqrestore(&ioapic_lock, flags); + + __io_apic_intr_restore; } static void unmask_IO_APIC_irq (unsigned int irq) { unsigned long flags; + __io_apic_intr_save; + spin_lock_irqsave(&ioapic_lock, flags); __unmask_IO_APIC_irq(irq); spin_unlock_irqrestore(&ioapic_lock, flags); + + __io_apic_intr_restore; } void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) @@ -1146,6 +1202,7 @@ return current_vector; } + static struct hw_interrupt_type ioapic_level_type; static struct hw_interrupt_type ioapic_edge_type; @@ -1230,7 +1287,14 @@ continue; if (IO_APIC_IRQ(irq)) { - vector = assign_irq_vector(irq); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + if (resetup_IO_APIC_flag == 0) + vector = assign_irq_vector(irq); + else + vector = reassign_irq_vector(irq); +#else + vector = assign_irq_vector(irq); +#endif entry.vector = vector; ioapic_register_intr(irq, vector, IOAPIC_AUTO); @@ -2376,6 +2440,371 @@ device_initcall(ioapic_init_sysfs); +/* append for FJe Linux at May 21, 2004. Sun Zhitai. */ +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +static int __init +reassign_irq_vector(int i) +{ + struct IO_APIC_route_entry entry; + int apic, pin, irq, vec; + unsigned long flags; + + for (irq=0; irq old apic:0, 0x%lx; 1, 0x%x", *(((int*)&entry) + 0), *(((int*)&entry) + 1)); +#endif /* DEBUG_CT */ + /* set on the irq's polarity: */ + entry.trigger = 1; + entry.polarity = !(entry.polarity); + spin_lock_irqsave(&ioapic_lock, flags); + io_apic_write(0, 0x10 + 2 * pin, *(((int *)&entry) + 0)); + io_apic_write(0, 0x11 + 2 * pin, *(((int *)&entry) + 1)); + *(((int*)&entry) + 0) = io_apic_read(0, 0x10 + 2 * pin); + *(((int*)&entry) + 1) = io_apic_read(0, 0x11 + 2 * pin); + spin_unlock_irqrestore(&ioapic_lock, flags); +for (i=0; i<1000; i++); +#ifdef DEBUG_CT +printk("raise-> New apic:0, 0x%lx; 1, 0x%x", *(((int*)&entry) + 0), *(((int*)&entry) + 1)); +#endif /* DEBUG_CT */ + + entry.polarity = !(entry.polarity); + spin_lock_irqsave(&ioapic_lock, flags); + io_apic_write(0, 0x10 + 2 * pin, *(((int *)&entry) + 0)); + io_apic_write(0, 0x11 + 2 * pin, *(((int *)&entry) + 1)); + *(((int*)&entry) + 0) = io_apic_read(0, 0x10 + 2 * pin); + *(((int*)&entry) + 1) = io_apic_read(0, 0x11 + 2 * pin); + spin_unlock_irqrestore(&ioapic_lock, flags); +for (i=0; i<1000; i++); + +#ifdef DEBUG_CT +printk("raise-> apic:0, 0x%lx; 1, 0x%x", *(((int*)&entry) + 0), *(((int*)&entry) + 1)); + print_IO_APIC(); +#endif /* DEBUG_CT */ +} + +void +kern_down_irq(int irq) +{ + struct IO_APIC_route_entry entry; + unsigned long flags,i; + int pin = irq_2_pin[irq].pin; + + /* Check delivery_mode to be sure we're not clearing an SMI pin */ + spin_lock_irqsave(&ioapic_lock, flags); + *(((int*)&entry) + 0) = io_apic_read(0, 0x10 + 2 * pin); + *(((int*)&entry) + 1) = io_apic_read(0, 0x11 + 2 * pin); + spin_unlock_irqrestore(&ioapic_lock, flags); + +#ifdef DEBUG_CT +printk("down-> old apic:0, 0x%lx; 1, 0x%x", *(((int*)&entry) + 0), *(((int*)&entry) + 1)); +#endif /* DEBUG_CT */ + + /* set off the irq's polarity: */ + entry.trigger = 0; + entry.polarity = ~entry.polarity; + spin_lock_irqsave(&ioapic_lock, flags); + io_apic_write(0, 0x10 + 2 * pin, *(((int *)&entry) + 0)); + io_apic_write(0, 0x11 + 2 * pin, *(((int *)&entry) + 1)); + *(((int*)&entry) + 0) = io_apic_read(0, 0x10 + 2 * pin); + *(((int*)&entry) + 1) = io_apic_read(0, 0x11 + 2 * pin); + spin_unlock_irqrestore(&ioapic_lock, flags); +for (i=0; i<1000; i++); +#ifdef DEBUG_CT +printk("down-> New apic:0, 0x%lx; 1, 0x%x", *(((int*)&entry) + 0), *(((int*)&entry) + 1)); +#endif /* DEBUG_CT */ + entry.polarity = ~entry.polarity; + spin_lock_irqsave(&ioapic_lock, flags); + io_apic_write(0, 0x10 + 2 * pin, *(((int *)&entry) + 0)); + io_apic_write(0, 0x11 + 2 * pin, *(((int *)&entry) + 1)); + *(((int*)&entry) + 0) = io_apic_read(0, 0x10 + 2 * pin); + *(((int*)&entry) + 1) = io_apic_read(0, 0x11 + 2 * pin); + spin_unlock_irqrestore(&ioapic_lock, flags); +for (i=0; i<1000; i++); +#ifdef DEBUG_CT +printk("down-> apic:0, 0x%lx; 1, 0x%x", *(((int*)&entry) + 0), *(((int*)&entry) + 1)); +#endif /* DEBUG_CT */ +} +EXPORT_SYMBOL(kern_raise_irq); +EXPORT_SYMBOL(kern_down_irq); + +#endif /* DEBUG_PRIORITY_VECTOR_CT */ + +extern void spurious_timer_interrupt(void); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + +asmlinkage void smp_spurious_timer_interrupt(void) +{ + ack_APIC_irq(); + irq_enter(); + printk("Spurious timer interrupt!!"); + irq_exit_nobh(); +} + + +static void __init +resetup_local_timer(void) +{ + unsigned int orig_lvtt_value,lvtt_value, new_lvtt_vec; + + orig_lvtt_value = apic_read(APIC_LVTT); + new_lvtt_vec = highest_pri_vec - 1; + if (new_lvtt_vec == 0x80) + new_lvtt_vec -= 1; + lvtt_value= (orig_lvtt_value & (~((unsigned int)0xff))) | new_lvtt_vec; + /* If user does not use privileged interrupt, local apic timer vector is + * set as same as LOCAL_TIMER_VECTOR. + */ +#ifdef DEBUG_PRIORITY_VECTOR + set_intr_gate(LOCAL_TIMER_VECTOR, spurious_timer_interrupt); +#endif /* DEBUG_PRIORITY_VECTOR */ + set_intr_gate(new_lvtt_vec, apic_timer_interrupt); + /* Set up Local APIC */ + apic_write_around(APIC_LVTT, lvtt_value); +} +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ +#if defined(CONFIG_PRIORITY_INTR) +void __init +resetup_vector(void) +{ + int irq,lvl,msk,vec, norm_lvl; + int i,j,odd=0; + + extern void print_IO_APIC(void); + + + memset(&sl_hmask_irq,0,sizeof(sl_hmask_irq)); + memset(&hmask2smask, 0, sizeof(hmask2smask)); + memset(&smask2hmask, 0, sizeof(smask2hmask)); + + + irq2lvl[0] = MAX_PRIO_LEVEL; + for(i=0;i sl_hmask_irq[j].irq) + { + lvl = sl_hmask_irq[i].soft_level; + irq = sl_hmask_irq[i].irq; + sl_hmask_irq[i].soft_level = sl_hmask_irq[j].soft_level; + sl_hmask_irq[i].irq = sl_hmask_irq[j].irq; + sl_hmask_irq[j].soft_level = lvl; + sl_hmask_irq[j].irq = irq; + } + } + } + } + +/* set vector and hardware mask */ + lvl = FIRST_PRIO_INTR_VECTOR; /* the first interrupt vector */ + for(i=0; i= MIN_PRIO_LEVEL) && + (sl_hmask_irq[i].soft_level <= MAX_PRIO_LEVEL) ) + { + if (odd) + { + if ((lvl & 0xf) != 0xf) + lvl = (lvl & 0xf0) - 1; + odd = 0; + } + + sl_hmask_irq[i].vector = lvl; + if (highest_pri_vec==0xff) + { + highest_pri_vec = lvl; + sl_hmask_irq[i].hard_mask = lvl; + lvl -= 1; /* to save a vector to local timer */ + if (lvl == 0x80) + lvl -= 1; + } + else if (sl_hmask_irq[i].soft_level == sl_hmask_irq[i-1].soft_level ) + { + sl_hmask_irq[i].hard_mask = sl_hmask_irq[i-1].hard_mask; + } + else + { + if ((lvl & 0xf) != 0xf) + { + lvl = (lvl & 0xf0) - 1; + sl_hmask_irq[i].vector = lvl; + } + sl_hmask_irq[i].hard_mask = lvl; + } + lvl -= 1; + if (lvl == 0x80) + lvl -= 1; + } + else if( (sl_hmask_irq[i].soft_level < MIN_NORM_LEVEL) || + (sl_hmask_irq[i].soft_level > MAX_NORM_LEVEL) ) + panic("illegal interrupt levels: %d\n", sl_hmask_irq[i].soft_level); + hmask2smask[sl_hmask_irq[i].vector] = sl_hmask_irq[i].soft_level; + smask2hmask[sl_hmask_irq[i].soft_level] = sl_hmask_irq[i].hard_mask; + } + norm_lvl = lvl; + if ((norm_lvl & 0xf) != 0xf) + norm_lvl = (norm_lvl & 0xf0) - 1; +/* sort with irq in small order */ + for (i=0; i= MIN_NORM_LEVEL) && + (sl_hmask_irq[i].soft_level <= MAX_NORM_LEVEL) && + (irq_vector[i] > 0 )) + { + sl_hmask_irq[i].vector = norm_lvl; + norm_lvl -= 1; + if (norm_lvl == 0x80) + norm_lvl -= 1; + } + hmask2smask[sl_hmask_irq[i].vector] = sl_hmask_irq[i].soft_level; + smask2hmask[sl_hmask_irq[i].soft_level] = sl_hmask_irq[i].hard_mask; + } + memset(&irq_vector, 0, sizeof(irq_vector)); /* reset irq_vector */ + +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + resetup_local_timer(); + reassign_irq_vector(0); +#endif + +#ifdef DEBUG_PRIORITY_VECTOR + deb_out(); /* show altered vectors */ + print_IO_APIC(); +#endif /* DEBUG_PRIORITY_VECTOR */ +} +#endif /* CONFIG_PRIORITY_INTR */ + /* -------------------------------------------------------------------------- ACPI-based IOAPIC Configuration -------------------------------------------------------------------------- */ diff -Naur linux-2.6.9/arch/i386/kernel/irq.c linux-2.6.9-lvlintr/arch/i386/kernel/irq.c --- linux-2.6.9/arch/i386/kernel/irq.c 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/irq.c 2004-12-27 14:23:45.000000000 +0900 @@ -34,6 +34,10 @@ #include #include #include +#if defined(CONFIG_PRIORITY_INTR) +#include +#endif +#include #include #include @@ -45,6 +49,10 @@ #include #include +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +extern int reassign_flag; +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ + /* * Linux has a controller-independent x86 interrupt architecture. * every controller has a 'controller-template', that is used @@ -222,6 +230,11 @@ int status = 1; /* Force the "do bottom halves" bit */ int ret, retval = 0; +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + if (likely(reassign_flag)) + set_hard_lvl_mask(irq, regs, action); + else +#endif /* CONFIG_PRIORITY_INTR */ if (!(action->flags & SA_INTERRUPT)) local_irq_enable(); @@ -234,6 +247,11 @@ } while (action); if (status & SA_SAMPLE_RANDOM) add_interrupt_randomness(irq); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + if (likely(reassign_flag)) + restore_hard_lvl_mask(irq, regs, action); + else +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ local_irq_disable(); return retval; } @@ -439,11 +457,18 @@ { long esp; +#ifdef CONFIG_PRIO_INTR_DIAG_TRACE_TPRI + void show_lvlmask_trace(void); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + __asm__ __volatile__("andl %%esp,%0" : "=r" (esp) : "0" (THREAD_SIZE - 1)); if (unlikely(esp < (sizeof(struct thread_info) + STACK_WARN))) { printk("do_IRQ: stack overflow: %ld\n", esp - sizeof(struct thread_info)); +#ifdef CONFIG_PRIO_INTR_DIAG_TRACE_TPRI + show_lvlmask_trace(); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ dump_stack(); } } @@ -489,6 +514,15 @@ * useful for irq hardware that does not mask cleanly in an * SMP environment. */ +#if defined(CONFIG_PRIORITY_NORMAL) + if ( likely(reassign_flag) && ( likely(!is_hard_masking_interrupt(irq))) ) + { + spin_unlock(&desc->lock); + wakeup_interrupt_thread(irq); + goto out2; /* unmask irq later */ + } +#endif + #ifdef CONFIG_4KSTACKS for (;;) { @@ -569,8 +603,12 @@ desc->handler->end(irq); spin_unlock(&desc->lock); +#if defined(CONFIG_PRIORITY_NORMAL) +out2: +#endif /* CONFIG_PRIORITY_NORMAL */ irq_exit(); + return 1; } @@ -692,7 +730,14 @@ return; desc = irq_desc + irq; - spin_lock_irqsave(&desc->lock,flags); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + /* Save the TPR simultaneously in prioritized interruption + * processing. + */ + spin_lock_interruptsave(&desc->lock,flags); +#else + spin_lock_irqsave(&desc->lock,flags); +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ p = &desc->action; for (;;) { struct irqaction * action = *p; @@ -948,7 +993,14 @@ /* * The following block of code has to be executed atomically */ +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + /* Save the TPR simultaneously in prioritized interruption + * processing. + */ + spin_lock_interruptsave(&desc->lock,flags); +#else spin_lock_irqsave(&desc->lock,flags); +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ p = &desc->action; if ((old = *p) != NULL) { /* Can't share interrupts unless both agree to */ diff -Naur linux-2.6.9/arch/i386/kernel/lvlintr386.c linux-2.6.9-lvlintr/arch/i386/kernel/lvlintr386.c --- linux-2.6.9/arch/i386/kernel/lvlintr386.c 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/lvlintr386.c 2004-12-27 14:23:45.000000000 +0900 @@ -0,0 +1,206 @@ +/* + * arch/i386/kernel/lvlintr386.c + * this is ported from PPC4xx. + * Author: Sun Zhitai(sun@cs.fujitsu.co.jp) + * + * 2004 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_MASK +#undef DEBUG_MASK +extern void show_registers(struct pt_regs *regs); +extern int reassign_flag; +asmlinkage unsigned long get_hardmask_level(void); +asmlinkage void remember_hardmask_level(struct pt_regs regs); +asmlinkage void recover_hardmask_level(struct pt_regs regs); + +/**********************************************************************/ +/* function: set_hardware_mask */ +/* Input: current_level ... current interrupt level. */ +/* new_level ... new interrupt level to be set. */ +/* Output: None */ +/* Return: Non-zero ... hardware mask level which is set. */ +/* 0 .... hardware mask is not set. */ +/* Note: None. */ +/* description: Set hardware mask level */ +/* author: Sun Zhitai */ +/* Revision: V1.0(original), March 22, 2004 */ +/* V1.1, ported for i386, by Sun Zhitai at March 26, 2004 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +set_hardware_mask(unsigned long current_level,unsigned long new_level) +{ + unsigned long flags; + unsigned long hwmsk; + unsigned long hard_level; + + local_interrupt_save(flags); + hard_level=lvl2hwpri(new_level); + +#ifdef CONFIG_PRIORITY_PRIVILEGED_IRQ + if (hard_level == MAX_PRIVILEGED_HWMSK) { + /* off IF of EFLAGS, disable interrupt */ + flags &= (~(X86_EFLAGS_IF)); + } else { +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ + hwmsk = hard_level<<(__MSK_EFLAGS_SHIFT); + flags = hwmsk|(flags & (__PURE_EFLAGS_MSK)); +#ifdef CONFIG_PRIORITY_PRIVILEGED_IRQ + } +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ + local_interrupt_restore(flags); + + return hard_level; +} +/**********************************************************************/ +/* function: get_hardmask_level */ +/* Input: None */ +/* Output: None */ +/* Return: Non-zero ... current hardware mask level in TPR. */ +/* 0 .... hardware mask is not set. */ +/* Note: None. */ +/* description: Get hardware mask level */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* V1.1, ported for i386, by Sun Zhitai at March 26, 2004 */ +/**********************************************************************/ +asmlinkage unsigned long +get_hardmask_level(void) +{ + unsigned long flags; + unsigned long mask = 0; + if (likely(reassign_flag)) { + __kern_intr_save(flags); + mask = apic_read(APIC_TASKPRI); + __kern_intr_restore(flags); + } + return (mask); +} + +/**********************************************************************/ +/* function: remember_hardmask_level */ +/* Input: regs, the context of INT,EXCEPT,SYSCAL */ +/* Output: None */ +/* Return: None */ +/* Note: This function assume the function is entried with IE-off.*/ +/* description: remember hardware mask level */ +/* author: Sun Zhitai */ +/* Revision: V1.0(original), May 28, 2004 */ +/**********************************************************************/ +asmlinkage void +remember_hardmask_level(struct pt_regs regs) +{ + unsigned long mask = 0; + if (likely(reassign_flag)) { + mask = apic_read(APIC_TASKPRI); + } + regs.xes = (regs.xes & 0xffff) | ((mask & 0xff) << 16); +} + +/**********************************************************************/ +/* function: check_remember_hardmask_level */ +/* Input: regs, the context of INT,EXCEPT,SYSCAL */ +/* Output: None */ +/* Return: None */ +/* Note: This function assume the function is entried with IE-off.*/ +/* description: check remembered hardware mask level with INTR mode. */ +/* author: Sun Zhitai */ +/* Revision: V1.0(original), May 28, 2004 */ +/**********************************************************************/ +asmlinkage void +check_remember_hardmask_level(struct pt_regs regs) +{ +#ifdef DEBUG_MASK + int irq = regs.orig_eax & 0xff; + + if (sl_hmask_irq[irq].hard_mask > ((regs.xes & 0xff0000) >> 16)) return; + + if(regs.xes & 0xff0000) { + if ( (!(0x58000000 & regs.xes)) /* except EXCEPTION and PRIVILEGED */ + && (!hardirq_count())){ + printk("remembered XES:0x%x precnt:%08x irq:%03d, vec:0x%x\n", + regs.xes, + preempt_count(), + irq, + sl_hmask_irq[irq].hard_mask); + show_registers(®s); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + show_lvlmask_trace(); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + } + } +#endif /* DEBUG_MASK */ +} + +/**********************************************************************/ +/* function: recover_hardmask_level */ +/* Input: regs, the context of INT,EXCEPT,SYSCAL */ +/* Output: None */ +/* Return: None */ +/* Note: This function assume the function is entried with IE-off.*/ +/* description: recover hardware mask level */ +/* author: Sun Zhitai */ +/* Revision: V1.0(original), May 28, 2004 */ +/**********************************************************************/ +asmlinkage void +recover_hardmask_level(struct pt_regs regs) +{ + unsigned long mask = 0; + if (likely(reassign_flag)) { + mask = (regs.xes >> 16) & 0xff; + apic_write(APIC_TASKPRI,mask); + } +} + +/**********************************************************************/ +/* function: check_hardmask_level */ +/* Input: regs ... interrupt context */ +/* Output: None */ +/* Return: None */ +/* Note: This function assume the function is entried with IE-off.*/ +/* description: Check hardware mask level */ +/* author: Takeharu KATO */ +/* Revision: V1.0, ported for i386, by T. KATO at May 24, 2004 */ +/**********************************************************************/ +asmlinkage void +check_hardmask_level(struct pt_regs regs) +{ +#if defined(DEBUG_MASK) + unsigned long mask = 0; + int irq_count=hardirq_count(); + int irq = regs.orig_eax & 0xff; + + if (!reassign_flag) return; + if (!(regs.xes & 0x20000000)) return; + if (regs.xes & 0x10000000) return; + + mask = apic_read(APIC_TASKPRI); + if (sl_hmask_irq[irq].hard_mask > mask) return; + if (mask && !irq_count) { + printk("Return to user with level-masked(0:0x%lx)!!\n",(mask & 0xff)); + printk("OrigEAX: %lx, Vec:0x%x\n", regs.orig_eax, sl_hmask_irq[irq].hard_mask); + show_registers(®s); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + show_lvlmask_trace(); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + while(1); + } +#endif /* DEBUG_MASK */ + return; +} diff -Naur linux-2.6.9/arch/i386/kernel/timers/timer_pit.c linux-2.6.9-lvlintr/arch/i386/kernel/timers/timer_pit.c --- linux-2.6.9/arch/i386/kernel/timers/timer_pit.c 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/timers/timer_pit.c 2004-12-27 11:42:03.000000000 +0900 @@ -21,6 +21,17 @@ #include "do_timer.h" #include "io_ports.h" +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +#define __i8259_intr_save \ + unsigned long f; \ + __kern_intr_save(f) +#define __i8259_intr_restore \ + __kern_intr_restore(f) +#else /* !(defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ)) */ +#define __i8259_intr_save +#define __i8259_intr_restore +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ + static int count_p; /* counter in get_offset_pit() */ static int __init init_pit(char* override) @@ -165,6 +176,7 @@ extern spinlock_t i8253_lock; unsigned long flags; + __i8259_intr_save; spin_lock_irqsave(&i8253_lock, flags); outb_p(0x34,PIT_MODE); /* binary, mode 2, LSB/MSB, ch 0 */ udelay(10); @@ -172,6 +184,7 @@ udelay(10); outb(LATCH >> 8 , PIT_CH0); /* MSB */ spin_unlock_irqrestore(&i8253_lock, flags); + __i8259_intr_restore; } static int timer_resume(struct sys_device *dev) diff -Naur linux-2.6.9/arch/i386/kernel/tpri_trace.c linux-2.6.9-lvlintr/arch/i386/kernel/tpri_trace.c --- linux-2.6.9/arch/i386/kernel/tpri_trace.c 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/tpri_trace.c 2004-12-27 14:23:45.000000000 +0900 @@ -0,0 +1,194 @@ +/* + * arch/i386/kernel/tpri_trace.c + * + * Author: Takeharu KATO (tkato@cs.fujitsu.co.jp) + * SUN Zhitai (sun@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#include +#include +#include +#include +#include +#include + +static int trace_index=0; +static int trace_on=0; +static lvlmask_trace_t trace_log[MAX_TRACE]; +extern int highest_pri_vec; +static inline void log_trace(unsigned long , unsigned long ,unsigned long ,unsigned long ,unsigned long ); +static void diag_local_irq_disable(unsigned long ret_addr); +static void diag_local_irq_enable(unsigned long ret_addr); +static void diag_local_irq_save(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags); +static void diag_local_irq_restore(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags); +static void diag_local_interrupt_save(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags); +static void diag_local_interrupt_restore(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags); + +static inline void +log_trace(unsigned long caller, unsigned long op,unsigned long tpr,unsigned long arg1,unsigned long arg2) +{ + if (trace_on) { + trace_log[trace_index].caller=caller; + trace_log[trace_index].op=op; + trace_log[trace_index].tpr=tpr; + trace_log[trace_index].arg1=arg1; + trace_log[trace_index].arg2=arg2; + ++trace_index; + trace_index %= MAX_TRACE; + } +} +#define trace_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory") +#define trace_restore_flags(x) __kern_intr_restore(x) + +static void +diag_local_irq_disable(unsigned long ret_addr) +{ + unsigned long old_flags; + unsigned long log_flags; + unsigned long old_mask; + + local_save_flags(old_flags); + trace_irq_save(log_flags); + old_mask =( apic_read(APIC_TASKPRI) & (__TPRI_REG_MSK)); + old_flags=( (old_mask & (__TPRI_REG_MSK) ) << (__MSK_EFLAGS_SHIFT) | + ((old_flags)&(__PURE_EFLAGS_MSK))); + if (old_mask < highest_pri_vec) + log_trace(ret_addr, TRACE_LOCAL_IRQ_DISABLE_NUM,old_mask,old_flags,0xffffffff); + trace_restore_flags(log_flags); +} +static void +diag_local_irq_enable(unsigned long ret_addr) +{ + unsigned long old_flags; + unsigned long log_flags; + unsigned long old_mask; + + local_save_flags(old_flags); + trace_irq_save(log_flags); + old_mask =( apic_read(APIC_TASKPRI) & (__TPRI_REG_MSK)); + old_flags=( (old_mask & (__TPRI_REG_MSK) ) << (__MSK_EFLAGS_SHIFT) | + ((old_flags)&(__PURE_EFLAGS_MSK))); + if (old_mask == highest_pri_vec) + log_trace(ret_addr, TRACE_LOCAL_IRQ_ENABLE_NUM,old_mask,old_flags,0xffffffff); + trace_restore_flags(log_flags); +} +static void +diag_local_irq_save(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags) +{ + unsigned long tpr=apic_read(APIC_TASKPRI); + unsigned long log_flags; + unsigned long r; + + trace_irq_save(log_flags); + r = apic_read(APIC_TASKPRI); + if ( r < highest_pri_vec) + log_trace(ret_addr, TRACE_LOCAL_IRQ_SAVE_NUM,tpr,oldflags,newflags); + trace_restore_flags(log_flags); +} +static void +diag_local_irq_restore(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags) +{ + unsigned long tpr=apic_read(APIC_TASKPRI); + unsigned long log_flags; + unsigned long r; + trace_irq_save(log_flags); + r = (newflags >> (__MSK_EFLAGS_SHIFT)) & (__TPRI_REG_MSK); + if ( r < highest_pri_vec) + log_trace(ret_addr, TRACE_LOCAL_IRQ_RESTORE_NUM,tpr,oldflags,newflags); + trace_restore_flags(log_flags); +} +static void +diag_local_interrupt_save(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ + unsigned long tpr=apic_read(APIC_TASKPRI); + unsigned long log_flags; + + trace_irq_save(log_flags); + log_trace(ret_addr, TRACE_LOCAL_INTERRUPT_SAVE_NUM,tpr,oldflags,newflags); + trace_restore_flags(log_flags); +} +static void +diag_local_interrupt_restore(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ + unsigned long tpr=apic_read(APIC_TASKPRI); + unsigned long log_flags; + + trace_irq_save(log_flags); + if ( (newflags>>__MSK_EFLAGS_SHIFT) < highest_pri_vec) + log_trace(ret_addr, TRACE_LOCAL_INTERRUPT_RESTORE_NUM,tpr,oldflags,newflags); + trace_restore_flags(log_flags); +} +void +show_lvlmask_trace(void) +{ + int i; + const char *names[]={"NONE", + "local_irq_disable ", + "local_irq_enable ", + "local_irq_save ", + "local_irq_restore ", + "local_interrupt_save ", + "local_interrupt_restore"}; + trace_on=0; + printk("lvl mask log:index[%d]\n",trace_index); + for(i=0;i +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern int highest_pri_vec; +extern int hmask2smask[]; +extern unsigned long soft_mask_level; + +static void __linux_local_interrupt_save_ptr(unsigned long *f); +static void __linux_local_interrupt_restore(unsigned long f); +static void __linux_local_irq_save_ptr(unsigned long *f); +static void __linux_local_irq_restore(unsigned long f); +static void __linux_local_irq_disable(void); +static void __linux_local_irq_enable(void); +static int __linux_irqs_disabled(void); +static int __linux_have_irqs_been_enabled(struct pt_regs *regs, unsigned long flgs); + +static void __tpri_local_interrupt_save_ptr(unsigned long *f); +static void __tpri_local_interrupt_restore(unsigned long f); +static void __tpri_local_irq_save_ptr(unsigned long *f); +static void __tpri_local_irq_restore(unsigned long f); +static void __tpri_local_irq_disable(void); +static void __tpri_local_irq_enable(void); +static int __tpri_irqs_disabled(void); +static int __tpri_have_irqs_been_enabled(struct pt_regs *regs, unsigned long flgs); + + +static prio_intr_mask_ops_t __prio_intr_intr_mask_ops={ + "prio_intr ops", + __tpri_local_interrupt_save_ptr, + __tpri_local_interrupt_restore, + __tpri_local_irq_save_ptr, + __tpri_local_irq_restore, + __tpri_local_irq_disable, + __tpri_local_irq_enable, + __tpri_irqs_disabled, + __tpri_have_irqs_been_enabled +}; +static prio_intr_mask_ops_t __linux_intr_mask_ops={ + "linux ops", + __linux_local_interrupt_save_ptr, + __linux_local_interrupt_restore, + __linux_local_irq_save_ptr, + __linux_local_irq_restore, + __linux_local_irq_disable, + __linux_local_irq_enable, + __linux_irqs_disabled, + __linux_have_irqs_been_enabled +}; + +prio_intr_mask_ops_t *current_intr_mask_ops=&__linux_intr_mask_ops; +EXPORT_SYMBOL(current_intr_mask_ops); + +void +switch_to_prio_intr(void) +{ + unsigned long flags; + + __kern_intr_save(flags); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + current_intr_mask_ops=&__prio_intr_intr_mask_ops; +#endif + __kern_intr_restore(flags); +} +void +switch_to_linux(void) +{ + unsigned long flags; + + __kern_intr_save(flags); + current_intr_mask_ops=&__linux_intr_mask_ops; + __kern_intr_restore(flags); +} + +static void +__linux_local_interrupt_save_ptr(unsigned long *f) +{ + unsigned long ff; + __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (ff): /* no input */ :"memory"); + *f=ff; +} +static void +__linux_local_interrupt_restore(unsigned long f) +{ + __kern_intr_restore(f); +} +static void +__linux_local_irq_save_ptr(unsigned long *f) +{ + unsigned long ff; + __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (ff): /* no input */ :"memory"); + *f=ff; +} +static void +__linux_local_irq_restore(unsigned long f) +{ + __kern_intr_restore(f); +} +static void +__linux_local_irq_disable(void) +{ + __asm__ __volatile__("cli": : :"memory"); +} +static void +__linux_local_irq_enable(void) +{ + __asm__ __volatile__("sti": : :"memory"); +} +static int +__linux_irqs_disabled(void) +{ + unsigned long flags; + + local_save_flags(flags); + return (!(flags & X86_EFLAGS_IF)); +} +static int +__linux_have_irqs_been_enabled(struct pt_regs *regs, unsigned long flgs) +{ + return (regs->eflags & (X86_EFLAGS_IF|VM_MASK)); +} + +static void +__tpri_local_interrupt_save_ptr(unsigned long *f) +{ + unsigned long mask; + unsigned long ff; +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + unsigned long ret_addr,new_flags; +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + + local_save_flags(ff); + *f=ff; + __kern_intr_disable(); + + mask=apic_read(APIC_TASKPRI); + *f |= (((mask)&(__TPRI_REG_MSK)) << (__MSK_EFLAGS_SHIFT)); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + local_save_flags(new_flags); + new_flags |= (((mask)&(__TPRI_REG_MSK)) << (__MSK_EFLAGS_SHIFT)); + ret_addr=(unsigned long)__builtin_return_address(0); + + prio_intr_diag_local_interrupt_save(ret_addr,*f,new_flags); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ +} +static void +__tpri_local_interrupt_restore(unsigned long f) +{ + unsigned long r; +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + unsigned long ret_addr,old_flags,tpr,new_flags; +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + r = (f >> (__MSK_EFLAGS_SHIFT)) & (__TPRI_REG_MSK); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + local_save_flags(old_flags); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + __kern_intr_disable(); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + tpr=apic_read(APIC_TASKPRI); + old_flags = ( (tpr & (__TPRI_REG_MSK) ) << (__MSK_EFLAGS_SHIFT) | + ((old_flags)&(__PURE_EFLAGS_MSK))); + new_flags=f; + ret_addr=(unsigned long)__builtin_return_address(0); + prio_intr_diag_local_interrupt_restore(ret_addr, old_flags,new_flags); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + apic_write(APIC_TASKPRI,r); /* restore the TPR */ + soft_mask_level = hmask2smask[r]; + f &= (__PURE_EFLAGS_MSK); /* to set eflags register */ + __kern_intr_restore(f); +} +static void +__tpri_local_irq_save_ptr(unsigned long *f) +{ + unsigned long r,ff; +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + unsigned long ret_addr; +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + local_save_flags(ff); + *f = ff; + __kern_intr_disable(); + r = apic_read(APIC_TASKPRI); + + *f = (ff & (__PURE_EFLAGS_MSK)) | (r << (__MSK_EFLAGS_SHIFT)); + apic_write(APIC_TASKPRI,highest_pri_vec); + /* disable all but privilege */ + soft_mask_level=hmask2smask[highest_pri_vec]; +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + ret_addr=(unsigned long)__builtin_return_address(0); + prio_intr_diag_local_irq_save(ret_addr, (ff | (r << (__MSK_EFLAGS_SHIFT))),*f); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + __kern_intr_restore(ff); +} +static void +__tpri_local_irq_restore(unsigned long f) +{ + unsigned long r; +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + unsigned long ret_addr,old_flags,tpr,new_flags; +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + + r = (f >> (__MSK_EFLAGS_SHIFT)) & (__TPRI_REG_MSK); + +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + local_save_flags(old_flags); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + + __kern_intr_disable(); + +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + tpr=apic_read(APIC_TASKPRI); + old_flags = ( (tpr & (__TPRI_REG_MSK) ) << (__MSK_EFLAGS_SHIFT) | + ((old_flags)&(__PURE_EFLAGS_MSK))); + new_flags=f; + ret_addr=(unsigned long)__builtin_return_address(0); + prio_intr_diag_local_irq_restore(ret_addr, old_flags,new_flags); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + + f &= (__PURE_EFLAGS_MSK); /* to set eflags register */ + apic_write(APIC_TASKPRI,r); /* restore the TPR */ + soft_mask_level = hmask2smask[r]; + __kern_intr_restore(f); +} +static void +__tpri_local_irq_disable(void) +{ + unsigned long r; + +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + unsigned long ret_addr; + ret_addr=(unsigned long)__builtin_return_address(0); + prio_intr_diag_local_irq_disable(ret_addr); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + __kern_intr_save(r); + apic_write(APIC_TASKPRI,highest_pri_vec); + soft_mask_level=hmask2smask[highest_pri_vec]; + __kern_intr_restore(r); +} +static void +__tpri_local_irq_enable(void) +{ + unsigned long r; + +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + unsigned long ret_addr; + ret_addr=(unsigned long)__builtin_return_address(0); + prio_intr_diag_local_irq_enable(ret_addr); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + __kern_intr_save(r); + apic_write(APIC_TASKPRI,0); + soft_mask_level=hmask2smask[0]; + __kern_intr_restore(r); + __kern_intr_enable(); +} +static int +__tpri_irqs_disabled(void) +{ + unsigned long lvl; + unsigned long flags; + int ret=0; + + local_save_flags(flags); + __kern_intr_disable(); + lvl = apic_read(APIC_TASKPRI); + __kern_intr_restore(flags); + ret = (! ( ( (lvl & (__TPRI_REG_MSK)) == 0) && (flags & (X86_EFLAGS_IF) ) ) ); + return ret; +} +static int +__tpri_have_irqs_been_enabled(struct pt_regs *regs, unsigned long flgs) +{ + int ret=0; + unsigned long flags; + unsigned long msk; + + __kern_intr_save(flags); + msk = ( (regs->xes >> 16) & (__TPRI_REG_MSK) ); + ret=( (regs->eflags & flgs & ~X86_EFLAGS_IF) || + ((regs->eflags & X86_EFLAGS_IF) && (msk == 0)) ); + __kern_intr_restore(flags); + + return ret; +} +int +__get_tpr(void) +{ + return (apic_read(APIC_TASKPRI) & 0xff); +} diff -Naur linux-2.6.9/arch/i386/kernel/traps.c linux-2.6.9-lvlintr/arch/i386/kernel/traps.c --- linux-2.6.9/arch/i386/kernel/traps.c 2004-12-27 11:05:58.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/kernel/traps.c 2004-12-27 11:42:03.000000000 +0900 @@ -46,6 +46,9 @@ #include #include #include +#if defined(CONFIG_PRIORITY_INTR) +#include +#endif /* CONFIG_PRIORITY_INTR */ #include #include @@ -54,6 +57,7 @@ #include #include + #include "mach_traps.h" asmlinkage int system_call(void); @@ -93,6 +97,7 @@ asmlinkage void spurious_interrupt_bug(void); asmlinkage void machine_check(void); + static int kstack_depth_to_print = 24; struct notifier_block *i386die_chain; static spinlock_t die_notifier_lock = SPIN_LOCK_UNLOCKED; @@ -221,7 +226,9 @@ int in_kernel = 1; unsigned long esp; unsigned short ss; - +#if defined(CONFIG_PRIORITY_INTR) + unsigned int v; +#endif /* CONFIG_PRIORITY_INTR */ esp = (unsigned long) (®s->esp); ss = __KERNEL_DS; if (regs->xcs & 3) { @@ -241,6 +248,10 @@ regs->esi, regs->edi, regs->ebp, esp); printk("ds: %04x es: %04x ss: %04x\n", regs->xds & 0xffff, regs->xes & 0xffff, ss); +#if defined(CONFIG_PRIORITY_INTR) + v = apic_read(APIC_TASKPRI); + printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK); +#endif /* CONFIG_PRIORITY_INTR */ printk("Process %s (pid: %d, threadinfo=%p task=%p)", current->comm, current->pid, current_thread_info(), current); /* @@ -296,7 +307,7 @@ if (__get_user(file, (char **)(eip + 4)) || (unsigned long)file < PAGE_OFFSET || __get_user(c, file)) file = ""; - + show_stack(current,NULL); printk("------------[ cut here ]------------\n"); printk(KERN_ALERT "kernel BUG at %s:%d!\n", file, line); @@ -305,6 +316,7 @@ /* Here we know it was a BUG but file-n-line is unavailable */ bug: + show_stack(current,NULL); printk("Kernel BUG\n"); } @@ -448,6 +460,7 @@ == NOTIFY_STOP) \ return; \ do_trap(trapnr, signr, str, 1, regs, error_code, NULL); \ + return; \ } #define DO_VM86_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \ @@ -710,8 +723,13 @@ SIGTRAP) == NOTIFY_STOP) return; /* It's safe to allow irq's after DR6 has been saved */ - if (regs->eflags & X86_EFLAGS_IF) - local_irq_enable(); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + if (have_irqs_been_enabled(regs, X86_EFLAGS_IF)) + local_irq_enable(); +#else /* !(defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ)) */ + if (regs->eflags & X86_EFLAGS_IF) + local_irq_enable(); +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ /* Mask out spurious debug traps due to lazy DR7 setting */ if (condition & (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3)) { @@ -738,6 +756,7 @@ */ if ((regs->xcs & 3) == 0) goto clear_TF_reenable; + if ((tsk->ptrace & (PT_DTRACE|PT_PTRACED)) == PT_DTRACE) goto clear_TF; } @@ -749,6 +768,7 @@ info.si_errno = 0; info.si_code = TRAP_BRKPT; + /* If this is a kernel mode trap, save the user PC on entry to * the kernel, that's what the debugger can make sense of. */ diff -Naur linux-2.6.9/arch/i386/mm/fault.c linux-2.6.9-lvlintr/arch/i386/mm/fault.c --- linux-2.6.9/arch/i386/mm/fault.c 2004-12-27 11:05:59.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/i386/mm/fault.c 2004-12-27 11:42:03.000000000 +0900 @@ -22,6 +22,7 @@ #include #include + #include #include #include @@ -230,8 +231,13 @@ SIGSEGV) == NOTIFY_STOP) return; /* It's safe to allow irq's after cr2 has been saved */ - if (regs->eflags & (X86_EFLAGS_IF|VM_MASK)) - local_irq_enable(); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + if (have_irqs_been_enabled(regs, (X86_EFLAGS_IF|VM_MASK))) + local_irq_enable(); +#else + if (regs->eflags & (X86_EFLAGS_IF|VM_MASK)) + local_irq_enable(); +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ tsk = current; @@ -426,6 +432,7 @@ if (is_prefetch(regs, address, error_code)) return; + /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. diff -Naur linux-2.6.9/arch/ppc/kernel/entry.S linux-2.6.9-lvlintr/arch/ppc/kernel/entry.S --- linux-2.6.9/arch/ppc/kernel/entry.S 2004-12-27 11:05:56.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/kernel/entry.S 2004-12-27 14:23:45.000000000 +0900 @@ -198,6 +198,10 @@ #ifdef SHOW_SYSCALLS bl do_show_syscall #endif /* SHOW_SYSCALLS */ +#if defined(CONFIG_PRIO_INTR_DIAG) + bl prio_intr_diag_syscall_trace +prio_intr_diag_cont_syscall: +#endif /* CONFIG_PRIO_INTR_DIAG */ rlwinm r10,r1,0,0,18 /* current_thread_info() */ lwz r11,TI_LOCAL_FLAGS(r10) rlwinm r11,r11,0,~_TIFL_FORCE_NOERROR @@ -237,6 +241,10 @@ 30: LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */ SYNC MTMSRD(r10) +#if defined(CONFIG_PRIO_INTR_DIAG) + bl prio_intr_diag_syscall_trace_exit +prio_intr_diag_cont_syscall_exit: +#endif /* CONFIG_PRIO_INTR_DIAG */ lwz r9,TI_FLAGS(r12) andi. r0,r9,(_TIF_SYSCALL_TRACE|_TIF_SIGPENDING|_TIF_NEED_RESCHED) bne- syscall_exit_work @@ -272,6 +280,32 @@ li r3,0 b ret_from_syscall +#if defined(CONFIG_PRIO_INTR_DIAG) +prio_intr_diag_syscall_trace: + SAVE_NVGPRS(r1) + li r0,0xc00 + stw r0,TRAP(r1) + mr r3,r1 + bl prio_intr_diag_syscall_enter + lwz r0,GPR0(r1) /* Restore original registers */ + lwz r3,GPR3(r1) + lwz r4,GPR4(r1) + lwz r5,GPR5(r1) + lwz r6,GPR6(r1) + lwz r7,GPR7(r1) + lwz r8,GPR8(r1) + REST_NVGPRS(r1) + b prio_intr_diag_cont_syscall + +prio_intr_diag_syscall_trace_exit: + SAVE_NVGPRS(r1) + mr r14,r3 + mr r3,r1 + bl prio_intr_diag_syscall_exit + mr r3,r14 + REST_NVGPRS(r1) + b prio_intr_diag_cont_syscall_exit +#endif /* CONFIG_PRIO_INTR_DIAG */ /* Traced system call support */ syscall_dotrace: SAVE_NVGPRS(r1) @@ -641,6 +675,12 @@ /* interrupts are hard-disabled at this point */ restore: +#if defined(CONFIG_PRIO_INTR_DIAG) + mr r0,r3 + mr r3,r1 + bl prio_intr_diag_restore + mr r3,r0 +#endif /* CONFIG_PRIO_INTR_DIAG */ lwz r0,GPR0(r1) lwz r2,GPR2(r1) REST_4GPRS(3, r1) diff -Naur linux-2.6.9/arch/ppc/kernel/head_44x.S linux-2.6.9-lvlintr/arch/ppc/kernel/head_44x.S --- linux-2.6.9/arch/ppc/kernel/head_44x.S 2004-12-27 11:05:56.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/kernel/head_44x.S 2004-12-27 11:42:03.000000000 +0900 @@ -297,7 +297,11 @@ interrupt_base: /* Critical Input Interrupt */ +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + CRITICAL_EXCEPTION(0x0100, CriticalInput, CriticalInterruptException) +#else CRITICAL_EXCEPTION(0x0100, CriticalInput, UnknownException) +#endif /* CONFIG_PRIORITY_INTR */ /* Machine Check Interrupt */ #ifdef CONFIG_440A @@ -627,8 +631,6 @@ mfspr r10,SPRN_DBSR /* check single-step/branch taken */ andis. r10,r10,(DBSR_IC|DBSR_BT)@h beq+ 1f - andi. r0,r9,MSR_PR /* check supervisor */ - beq 2f /* branch if we need to fix it up... */ /* continue normal handling for a critical exception... */ 1: mfspr r4,SPRN_DBSR diff -Naur linux-2.6.9/arch/ppc/kernel/head_4xx.S linux-2.6.9-lvlintr/arch/ppc/kernel/head_4xx.S --- linux-2.6.9/arch/ppc/kernel/head_4xx.S 2004-12-27 11:05:56.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/kernel/head_4xx.S 2004-12-27 11:42:03.000000000 +0900 @@ -17,8 +17,6 @@ * Author: MontaVista Software, Inc. * frank_rowand@mvista.com or source@mvista.com * debbie_chu@mvista.com - * - * * Module name: head_4xx.S * * Description: @@ -284,8 +282,11 @@ /* * 0x0100 - Critical Interrupt Exception */ +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + CRITICAL_EXCEPTION(0x0100, CriticalInterrupt, CriticalInterruptException) +#else CRITICAL_EXCEPTION(0x0100, CriticalInterrupt, UnknownException) - +#endif /* * 0x0200 - Machine Check Exception */ @@ -727,7 +728,6 @@ beq+ 1f andi. r0,r9,MSR_IR|MSR_PR /* check supervisor + MMU off */ beq 2f /* branch if we need to fix it up... */ - /* continue normal handling for a critical exception... */ 1: mfspr r4,SPRN_DBSR addi r3,r1,STACK_FRAME_OVERHEAD diff -Naur linux-2.6.9/arch/ppc/kernel/head_booke.h linux-2.6.9-lvlintr/arch/ppc/kernel/head_booke.h --- linux-2.6.9/arch/ppc/kernel/head_booke.h 2004-12-27 11:05:56.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/kernel/head_booke.h 2004-12-27 11:42:03.000000000 +0900 @@ -194,8 +194,8 @@ CRITICAL_EXCEPTION_PROLOG; \ addi r3,r1,STACK_FRAME_OVERHEAD; \ EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ - NOCOPY, transfer_to_handler_full, \ - ret_from_except_full) + NOCOPY, crit_transfer_to_handler, \ + ret_from_crit_exc) #define MCHECK_EXCEPTION(n, label, hdlr) \ START_EXCEPTION(label); \ diff -Naur linux-2.6.9/arch/ppc/kernel/irq.c linux-2.6.9-lvlintr/arch/ppc/kernel/irq.c --- linux-2.6.9/arch/ppc/kernel/irq.c 2004-12-27 11:05:56.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/kernel/irq.c 2004-12-27 14:23:45.000000000 +0900 @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include #include @@ -57,7 +59,6 @@ #include #include #include - #define NR_MASK_WORDS ((NR_IRQS + 31) / 32) extern atomic_t ipi_recv; @@ -129,6 +130,11 @@ unsigned long flags; struct irqaction *old, **p; irq_desc_t *desc = irq_desc + irq; +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + extern unsigned long critical_interrupts_map[]; + int word = irq >> 5; + int bit = irq & 0x1f; +#endif /* * Some drivers like serial.c use request_irq() heavily, @@ -150,11 +156,23 @@ /* * The following block of code has to be executed atomically */ +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + /* We assume spin_lock_irqsave(local_irq_save) saves msr including + * CE bit. + */ + spin_lock_interruptsave(&desc->lock,flags); +#else spin_lock_irqsave(&desc->lock,flags); +#endif /* CONFIG_PRIORITY_INTR */ p = &desc->action; if ((old = *p) != NULL) { /* Can't share interrupts unless both agree to */ - if (!(old->flags & new->flags & SA_SHIRQ)) { + if ( +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + ( critical_interrupts_map[word] & (1<<(31 - bit)) ) || +#endif + (!(old->flags & new->flags & SA_SHIRQ)) ) { + spin_unlock_irqrestore(&desc->lock,flags); return -EBUSY; } @@ -192,7 +210,14 @@ unsigned long flags; desc = irq_desc + irq; +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + /* We assume spin_lock_irqsave(local_irq_save) saves msr including + * CE bit. + */ + spin_lock_interruptsave(&desc->lock,flags); +#else spin_lock_irqsave(&desc->lock,flags); +#endif /* CONFIG_PRIORITY_INTR */ p = &desc->action; for (;;) { struct irqaction * action = *p; @@ -411,14 +436,23 @@ return 0; } +#if defined(CONFIG_PRIORITY_INTR) +void +handle_irq_event(int irq, struct pt_regs *regs, struct irqaction *action) +#else static inline void handle_irq_event(int irq, struct pt_regs *regs, struct irqaction *action) +#endif { int status = 0; int ret; - if (!(action->flags & SA_INTERRUPT)) - local_irq_enable(); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + set_hard_lvl_mask(irq, regs, action); +#else + if (!(action->flags & SA_INTERRUPT)) + local_irq_enable(); +#endif do { ret = action->handler(irq, action->dev_id, regs); @@ -428,7 +462,11 @@ } while (action); if (status & SA_SAMPLE_RANDOM) add_interrupt_randomness(irq); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + restore_hard_lvl_mask(irq, regs, action); +#else local_irq_disable(); +#endif /* CONFIG_PRIORITY_INTR */ } /* @@ -496,6 +534,13 @@ * useful for irq hardware that does not mask cleanly in an * SMP environment. */ +#if defined(CONFIG_PRIORITY_NORMAL) + if ( likely(!is_hard_masking_interrupt(irq)) ) { + wakeup_interrupt_thread(irq); + goto out2; /* unmask irq later */ + } +#endif + for (;;) { spin_unlock(&desc->lock); handle_irq_event(irq, regs, action); @@ -517,6 +562,9 @@ else if (irq_desc[irq].handler->enable) irq_desc[irq].handler->enable(irq); } +#if defined(CONFIG_PRIORITY_NORMAL) + out2: +#endif spin_unlock(&desc->lock); } @@ -525,6 +573,8 @@ int irq, first = 1; irq_enter(); + prio_intr_diag_irq_enter(regs); + /* * Every platform is required to implement ppc_md.get_irq. * This function will either return an irq number or -1 to @@ -540,7 +590,9 @@ if (irq != -2 && first) /* That's not SMP safe ... but who cares ? */ ppc_spurious_interrupts++; - irq_exit(); + + prio_intr_diag_irq_exit(regs); + irq_exit(); } unsigned long probe_irq_on (void) diff -Naur linux-2.6.9/arch/ppc/kernel/misc.S linux-2.6.9-lvlintr/arch/ppc/kernel/misc.S --- linux-2.6.9/arch/ppc/kernel/misc.S 2004-12-27 11:05:56.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/kernel/misc.S 2004-12-27 11:42:03.000000000 +0900 @@ -273,6 +273,7 @@ #endif /* CONFIG_CPU_FREQ_PMAC && CONFIG_6xx */ +#if !defined(CONFIG_PRIORITY_INTR) /* void local_save_flags_ptr(unsigned long *flags) */ _GLOBAL(local_save_flags_ptr) mfmsr r4 @@ -374,7 +375,7 @@ _GLOBAL(local_irq_enable) mfmsr r3 /* Get current state */ - ori r3,r3,MSR_EE /* Turn on 'EE' bit */ + ori r3,r3,MSR_EE SYNC /* Some chip revs have problems here... */ mtmsr r3 /* Update machine state */ blr @@ -400,7 +401,7 @@ nop nop _GLOBAL(local_irq_enable_end) - +#endif /* !defined(CONFIG_PRIORITY_INTR) */ /* * complement mask on the msr then "or" some values on. * _nmask_and_or_msr(nmask, value_to_or) diff -Naur linux-2.6.9/arch/ppc/platforms/4xx/Kconfig linux-2.6.9-lvlintr/arch/ppc/platforms/4xx/Kconfig --- linux-2.6.9/arch/ppc/platforms/4xx/Kconfig 2004-12-27 11:05:57.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/platforms/4xx/Kconfig 2004-12-27 14:23:45.000000000 +0900 @@ -214,4 +214,87 @@ bool depends on SERIAL_SICC && UART0_TTYS1 default y + +# for prioritized interrupt processing +config PRIORITY_INTR + depends on (EXPERIMENTAL && !SMP) && (PREEMPT) + bool "Level based interruption handling" + default n + ---help--- + This adopt interrupt priorities for interrupt handlers + (top half handlers) + to handle interruption orderly(like Solaris). + Current version of this feature does not support SMP Kernel. + +config PRIORITY_INTR_ASSIGN + string "Level assignments" + depends on PRIORITY_INTR + default "" + ---help--- + Prioritized interruption handling facility makes top halves of + kernel threads run in SCHED_FIFO class(except the highest priority + (512), it's called directly as normal linux kernel, not threaded.). + And priority levels of interrupts are expressed as + thread's priority. + Please set interrupts' priority level as following: + IRQ`:'Prioriy,IRQ`:'Prioriy,IRQ`:'Prioriy,...,IRQ`:'Prioriy + Range of priority=1(least significant)..512(most significant) + e.g. To assign IRQ1 as priority 10 and IRQ3 as 12: + 1:10,3:12 + Valid interrupt priority and type of interrupts are described + as follow: + 512 Quick interrupts + 385 -- 511 Reserved for quick interrupts + 257 -- 384 Priority interrupts(included reserved values.) + 129 -- 256 Normal interrupts or softirq + 100 -- 128 Reserved for normal interrupts or softirq + + We recommend configure levels of normal interrupts + between 129 and 256,but you can configure it between 1 and 256 + as you need. + +config PRIORITY_INTR_DEFAULT_LVL + int "Default Level" + range 129 384 + depends on PRIORITY_INTR + default "129" + ---help--- + This is default priority level for top halves. + +config PRIORITY_INTR_NORMAL + bool "Normal interrupt facility " + depends on PRIORITY_INTR + ---help--- + This facility make use of interrupt threads. + +config PRIORITY_PRIO + bool "Hardware assist interupt masking " + depends on PRIORITY_INTR + ---help--- + This facility make use of interrupt threads. + +config PRIORITY_INTR_SOFTIRQ_LVL + int "Priority level of softirqs" + range 1 256 + depends on (PRIORITY_INTR && PRIORITY_INTR_NORMAL) + default "100" + ---help--- + This is default priority level for softirqs. + In general, it should be set a value lower than top halves, and + it is preferable to set it as higher than user processes which + runs on your system. We recommend configure this value + between 129 and 256 and lower than normal interrupts priority + level, but you can configure this value between 1 and 256 as you + need. + +config PRIORITY_PRIVILEGED_IRQ + bool "Privileged interruption handling facility" + depends on PRIORITY_INTR + default n + ---help--- + Quick interrupt handling facility for embedded systems. + If this item is enabled,accepting some interrupts in current + Linux kernel's critical sections. If you set this option, + the highest priority interrupts are treated as quick interruption. + endmenu diff -Naur linux-2.6.9/arch/ppc/platforms/4xx/Makefile linux-2.6.9-lvlintr/arch/ppc/platforms/4xx/Makefile --- linux-2.6.9/arch/ppc/platforms/4xx/Makefile 2004-12-27 11:05:57.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/platforms/4xx/Makefile 2004-12-27 11:42:03.000000000 +0900 @@ -21,3 +21,4 @@ obj-$(CONFIG_440GX) += ibm440gx.o obj-$(CONFIG_405EP) += ibm405ep.o obj-$(CONFIG_405GPR) += ibm405gpr.o +obj-$(CONFIG_PRIORITY_INTR) += crit_intr.o diff -Naur linux-2.6.9/arch/ppc/platforms/4xx/crit_intr.c linux-2.6.9-lvlintr/arch/ppc/platforms/4xx/crit_intr.c --- linux-2.6.9/arch/ppc/platforms/4xx/crit_intr.c 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/arch/ppc/platforms/4xx/crit_intr.c 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,391 @@ +/* + * arch/ppc/platforms/4xx/crit_intr.c + * + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#include +#include +#include +#include +#include +#include +#include +#undef CRIT_INTR_DEBUG + +unsigned long critical_interrupts_map[NR_IRQS/(sizeof(long)*8)]; +static unsigned long hard_level=0; +static inline void update_critical_interrupt_state(int uicno); +static inline void unset_irq_critical(int irq); + +#if !defined(UIC1) +#define UIC1 UIC0 +#endif + +/**********************************************************************/ +/* function: update_critical_interrupt_state */ +/* Input: int uicno ... Number of UIC to be set */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Set the CR bit states into UIC */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static inline void +update_critical_interrupt_state(int uicno) +{ + switch(uicno) { + case 0: + mtdcr(DCRN_UIC_CR(UIC0), critical_interrupts_map[uicno]); + break; + case 1: + mtdcr(DCRN_UIC_CR(UIC1), critical_interrupts_map[uicno]); + break; + } +} + +/**********************************************************************/ +/* function: ack_critical_interrupt */ +/* Input: int irq ... irq number to ack */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Send ack to UIC */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Oct 21, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static inline void +ack_critical_interrupt(int irq) +{ + int bit, word; + + bit = irq & 0x1f; + word = irq >> 5; + + ppc_cached_irq_mask[word] &= ~(1 << (31 - bit)); + switch (word) { + case 0: + mtdcr(DCRN_UIC_ER(UIC0), ppc_cached_irq_mask[word]); + mtdcr(DCRN_UIC_SR(UIC0), (1 << (31 - bit))); + break; +#if NR_UICS > 1 + case 1: + mtdcr(DCRN_UIC_ER(UIC1), ppc_cached_irq_mask[word]); + mtdcr(DCRN_UIC_SR(UIC1), (1 << (31 - bit))); + mtdcr(DCRN_UIC_SR(UIC0), (1 << (31 - UIC0_UIC1NC))); + break; +#endif + } +} + +/**********************************************************************/ +/* function: end_critical_interrupt */ +/* Input: int irq ... irq number to ack */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: end critical interrupt */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Oct 21, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static inline void +end_critical_interrupt(int irq) +{ + int bit, word; + unsigned int tr_bits; + + bit = irq & 0x1f; + word = irq >> 5; + + /* + * Check trigger type + */ + switch (word) { + case 0: + tr_bits = mfdcr(DCRN_UIC_TR(UIC0)); + break; + case 1: + tr_bits = mfdcr(DCRN_UIC_TR(UIC1)); + break; + } + + if ((tr_bits & (1 << (31 - bit))) == 0) { + /* level trigger */ + switch (word) { + case 0: + mtdcr(DCRN_UIC_SR(UIC0), 1 << (31 - bit)); + break; +#if NR_UICS > 1 + case 1: + mtdcr(DCRN_UIC_SR(UIC1), 1 << (31 - bit)); + /* ACK cascaded interrupt in UIC0 */ + mtdcr(DCRN_UIC_SR(UIC0), (1 << (31 - UIC0_UIC1NC))); + break; +#endif + } + } + ppc_cached_irq_mask[word] |= 1 << (31 - bit); + switch (word) { + case 0: + mtdcr(DCRN_UIC_ER(UIC0), ppc_cached_irq_mask[word]); + break; + case 1: + mtdcr(DCRN_UIC_ER(UIC1), ppc_cached_irq_mask[word]); + break; + } + +} + +/**********************************************************************/ +/* function: set_irq_critical */ +/* Input: int irq ... IRQ number of critical interrupt. */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Make the irq critical. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void +set_irq_critical(int irq) +{ + int word = irq >> 5; + int bit = irq & 0x1f; + long flags; + + local_interrupt_save(flags); + critical_interrupts_map[word]|=1<<(31 - bit); + update_critical_interrupt_state(word); + local_interrupt_restore(flags); +} + +/**********************************************************************/ +/* function: unset_irq_critical */ +/* Input: int irq ... IRQ number of critical interrupt. */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Make the irq NON-critical. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void +unset_irq_critical(int irq) +{ + int word = irq >> 5; + int bit = irq & 0x1f; + long flags; + + local_irq_save(flags); + critical_interrupts_map[word] &= ~(1<<(31 - bit)); + update_critical_interrupt_state(word); + local_interrupt_restore(flags); +} +/**********************************************************************/ +/* function: ppc405_pic_get_critical_irq */ +/* Input: pt_regs *regs ... Interrupt context(No used on PPC405) */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Retrive IRQ Number. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Sep 22, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +ppc405_pic_get_critical_irq(struct pt_regs *regs) +{ + int irq, cas_irq; + unsigned long bits; +#ifdef CRIT_INTR_DEBUG + unsigned long orig_bits; +#endif /* CRIT_INTR_DEBUG */ + + cas_irq = 0; + /* + * Only report the status of those interrupts that are actually + * enabled. + */ + + bits = mfdcr(DCRN_UIC_MSR(UIC0)); +#ifdef CRIT_INTR_DEBUG + orig_bits=bits; +#endif /* CRIT_INTR_DEBUG */ + +#if (NR_UICS > 1) + if (bits & UIC_CASCADE_MASK) { +#ifdef CRIT_INTR_DEBUG + orig_bits = bits = mfdcr(DCRN_UIC_MSR(UIC1)); + bits &= critical_interrupts_map[1]; +#else + bits = (mfdcr(DCRN_UIC_MSR(UIC1)) & critical_interrupts_map[1]); +#endif /* CRIT_INTR_DEBUG */ + cas_irq = 32 - ffs(bits); + irq = 32 + cas_irq; + } else { + bits &= critical_interrupts_map[0]; + irq = 32 - ffs(bits); + if (irq == 32) + irq = -1; + } +#else + /* + * Walk through the interrupts from highest priority to lowest, and + * report the first pending interrupt found. + * We want PPC, not C bit numbering, so just subtract the ffs() + * result from 32. + */ + bits &= critical_interrupts_map[0]; + irq = 32 - ffs(bits); +#endif /* (NR_UICS > 1) */ + if (irq == (NR_UIC_IRQS * NR_UICS)) + irq = -1; + +#ifdef CRIT_INTR_DEBUG + printk(KERN_DEBUG "ppc405_pic_get_critical_irq - irq %d bit 0x%lx orig_bit 0x%lx\n", irq, bits,orig_bits); +#endif /* CRIT_INTR_DEBUG */ + + return (irq); +} + +/**********************************************************************/ +/* function: CriticalInterruptException */ +/* Input: struct pt_regs *regs ... Interrupt context */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Invoke critical interrupt handlers */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void +CriticalInterruptException(struct pt_regs *regs) +{ + int irq; + int status=0; + struct irqaction *action; + irq_desc_t *desc; + + while ((irq = get_critical_irq(regs)) >= 0) { + desc = irq_desc + irq; + /* IBM 4xx does not support SMP. + * So we need not to care SMP case here. + */ + ack_critical_interrupt(irq); //Ack and mask + /* Note: IRQ_REPLAY/IRQ_PENDING/IRQ_WAITING is never set on + * sigle processor environments. + */ + action = desc->action; + if (!action || !action->handler) { + /* There is no handler. + * This is spurious interrupt. + */ + /* We mask the interrupt not to raise. */ + ++desc->depth; + desc->status |= IRQ_DISABLED; + mask_irq(irq); + unset_irq_critical(irq); + goto out; + } + status |= IRQ_INPROGRESS; /* we are handling it */ + + desc->status = status; + handle_irq_event(irq, regs, action); + out: + desc->status &= ~IRQ_INPROGRESS; + /* + * The handler has to deal with interrupts which got + * disabled while the handler was running. + */ + end_critical_interrupt(irq); + } +} +/**********************************************************************/ +/* function: set_hardware_mask */ +/* Input: current_level ... current interrupt level. */ +/* new_level ... new interrupt level to be set. */ +/* Output: None */ +/* Return: Non-zero ... hardware mask level which is set. */ +/* 0 .... hardware mask is not set. */ +/* Note: None. */ +/* description: Set hardware mask level */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +set_hardware_mask(unsigned long current_level,unsigned long new_level) +{ + unsigned long flags; + + local_interrupt_save(flags); + hard_level=lvl2hwpri(new_level); + switch(hard_level) + { + case MAX_PRIVILEGED_HWMSK: + flags &= ~(MSR_EE|MSR_CE); + break; + case MAX_PRIO_HWMSK: + flags &= ((~MSR_EE)|MSR_CE); + break; + default: + flags |= (MSR_EE|MSR_CE); + break; + } + local_interrupt_restore(flags); + + return hard_level; +} +/**********************************************************************/ +/* function: get_hardmask_level */ +/* Input: None */ +/* Output: None */ +/* Return: Non-zero ... hardware mask level which is set. */ +/* 0 .... hardware mask is not set. */ +/* Note: None. */ +/* description: Set hardware mask level */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +unsigned long +get_hardmask_level(void) +{ + return (hard_level); +} + +/**********************************************************************/ +/* function: arch_setup_lvlintr */ +/* Input: None */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Set up architecture dependent issues. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Jun 6, 2004 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void +arch_setup_lvlintr(void) +{ + int i; + int lvl; + + for(i=0;i #include #include - +#include #include #include #include #include +#if defined(CONFIG_PRIORITY_INTR) +#include +#endif /* CONFIG_PRIORITY_INTR */ /* Global Variables */ struct hw_interrupt_type *ppc4xx_pic; /* @@ -108,8 +111,22 @@ bit = irq & 0x1f; word = irq >> 5; +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + { + long flags; + + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_INTR */ + ppc_cached_irq_mask[word] |= (1 << (31 - bit)); mtdcr(DCRN_EXIER, ppc_cached_irq_mask[word]); + +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ + } static void @@ -120,8 +137,18 @@ bit = irq & 0x1f; word = irq >> 5; + { + long flags; +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ ppc_cached_irq_mask[word] &= ~(1 << (31 - bit)); mtdcr(DCRN_EXIER, ppc_cached_irq_mask[word]); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ } static void @@ -132,9 +159,20 @@ bit = irq & 0x1f; word = irq >> 5; +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + { + long flags; + + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ ppc_cached_irq_mask[word] &= ~(1 << (31 - bit)); mtdcr(DCRN_EXIER, ppc_cached_irq_mask[word]); mtdcr(DCRN_EXISR, (1 << (31 - bit))); +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ } #else @@ -158,6 +196,25 @@ #ifdef UIC_DEBUG printk("ppc4xx_uic_enable - irq %d word %d bit 0x%x\n", irq, word, bit); #endif +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + { + long flags; + + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ + +#if defined(CONFIG_PRIO_INTR_DIAG) + { + long tflags; + unsigned long oldmask; + unsigned long lr; + + __in_diag_save_flags(&tflags); + oldmask=ppc_cached_irq_mask[word]; + __in_diag_restore_flags(&tflags); + +#endif /* CONFIG_PRIO_INTR_DIAG */ ppc_cached_irq_mask[word] |= 1 << (31 - bit); switch (word) { case 0: @@ -182,6 +239,17 @@ desc->status = desc->status & ~IRQ_LEVEL; break; } +#if defined(CONFIG_PRIO_INTR_DIAG) + __in_diag_save_flags(&tflags); + get_lr(lr); + prio_intr_diag_enable_irq(lr, irq, oldmask,ppc_cached_irq_mask[word], desc->depth); + __in_diag_restore_flags(&tflags); + } +#endif /* CONFIG_PRIO_INTR_DIAG */ +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ } @@ -196,6 +264,24 @@ printk("ppc4xx_uic_disable - irq %d word %d bit 0x%x\n", irq, word, bit); #endif +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + { + long flags; + + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ +#if defined(CONFIG_PRIO_INTR_DIAG) + { + unsigned long lr; + unsigned long oldmask; + long tflags; + irq_desc_t *desc = irq_desc + irq; + + __in_diag_save_flags(&tflags); + oldmask=ppc_cached_irq_mask[word]; + __in_diag_restore_flags(&tflags); +#endif /* CONFIG_PRIO_INTR_DIAG */ ppc_cached_irq_mask[word] &= ~(1 << (31 - bit)); switch (word) { case 0: @@ -208,6 +294,19 @@ mtdcr(DCRN_UIC_ER(UIC2), ppc_cached_irq_mask[word]); break; } +#if defined(CONFIG_PRIO_INTR_DIAG) + + + __in_diag_save_flags(&tflags); + get_lr(lr); + prio_intr_diag_disable_irq(lr, irq, oldmask, ppc_cached_irq_mask[word], desc->depth); + __in_diag_restore_flags(&tflags); + } +#endif /* CONFIG_PRIO_INTR_DIAG */ +#if defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_PRIO || CONFIG_PRIORITY_PRIVILEGED_IRQ */ } static void @@ -222,6 +321,25 @@ printk("ppc4xx_uic_disable_and_ack - irq %d word %d bit 0x%x\n", irq, word, bit); #endif +#if defined(CONFIG_PRIORITY_INTR) + { + long flags; + + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_INTR */ +#if defined(CONFIG_PRIO_INTR_DIAG) + { + unsigned long lr; + long tflags; + irq_desc_t *desc = irq_desc + irq; + + __in_diag_save_flags(&tflags); + get_lr(lr); + prio_intr_diag_enter_ack_irq(lr, irq, ppc_cached_irq_mask[word], desc->depth); + __in_diag_restore_flags(&tflags); + } +#endif /* CONFIG_PRIO_INTR_DIAG */ ppc_cached_irq_mask[word] &= ~(1 << (31 - bit)); switch (word) { case 0: @@ -249,7 +367,22 @@ #endif break; } - +#if defined(CONFIG_PRIO_INTR_DIAG) + { + unsigned long lr; + long tflags; + irq_desc_t *desc = irq_desc + irq; + + __in_diag_save_flags(&tflags); + get_lr(lr); + prio_intr_diag_exit_ack_irq(lr, irq, ppc_cached_irq_mask[word], desc->depth); + __in_diag_restore_flags(&tflags); + } +#endif /* CONFIG_PRIO_INTR_DIAG */ +#if defined(CONFIG_PRIORITY_INTR) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_INTR */ } static void @@ -303,7 +436,13 @@ break; } } - +#if defined(CONFIG_PRIORITY_INTR) + { + long flags; + + local_interrupt_save(flags); + /* Disable critical interrupt during mask operation. */ +#endif /* CONFIG_PRIORITY_INTR */ if (!(irq_desc[irq].status & (IRQ_DISABLED | IRQ_INPROGRESS))) { ppc_cached_irq_mask[word] |= 1 << (31 - bit); switch (word) { @@ -318,6 +457,23 @@ break; } } +#if defined(CONFIG_PRIO_INTR_DIAG) + { + unsigned long lr; + long tflags; + irq_desc_t *desc = irq_desc + irq; + + __in_diag_save_flags(&tflags); + get_lr(lr); + prio_intr_diag_exit_ack_irq(lr, irq, ppc_cached_irq_mask[word], desc->depth); + __in_diag_restore_flags(&tflags); + } +#endif /* CONFIG_PRIO_INTR_DIAG */ + +#if defined(CONFIG_PRIORITY_INTR) + local_interrupt_restore(flags); + } +#endif /* CONFIG_PRIORITY_INTR */ } static struct hw_interrupt_type ppc4xx_uic = { diff -Naur linux-2.6.9/include/asm-i386/bitops.h linux-2.6.9-lvlintr/include/asm-i386/bitops.h --- linux-2.6.9/include/asm-i386/bitops.h 2004-12-27 11:07:13.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/bitops.h 2004-12-27 11:42:03.000000000 +0900 @@ -388,17 +388,31 @@ * unlikely to be set. It's guaranteed that at least one of the 140 * bits is cleared. */ +#if defined(CONFIG_PRIORITY_INTR) +#include +#endif /* CONFIG_PRIORITY_INTR */ static inline int sched_find_first_bit(const unsigned long *b) { +#if defined(CONFIG_PRIORITY_INTR) + int i; +#endif /* CONFIG_PRIORITY_INTR */ if (unlikely(b[0])) return __ffs(b[0]); if (unlikely(b[1])) return __ffs(b[1]) + 32; if (unlikely(b[2])) return __ffs(b[2]) + 64; +#if defined(CONFIG_PRIORITY_INTR) + for(i=3;i<(MAX_PRIO>>5);++i) + if (b[i]) + return __ffs(b[i]) + (i<<5); + + return __ffs(b[(MAX_PRIO>>5)]) + ( (MAX_PRIO>>5)<<5); +#else if (b[3]) return __ffs(b[3]) + 96; return __ffs(b[4]) + 128; +#endif /* CONFIG_PRIORITY_INTR */ } /** diff -Naur linux-2.6.9/include/asm-i386/crit_intr.h linux-2.6.9-lvlintr/include/asm-i386/crit_intr.h --- linux-2.6.9/include/asm-i386/crit_intr.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/crit_intr.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,19 @@ +/* + * Critical Interrupt of i386 + */ + +#ifndef _ASM_CRIT_INTR_H_ +#define _ASM_CRIT_INTR_H_ +#include +#define PRIVILEGED_IRQ_FLAG (1<<28) +extern __inline__ int +get_critical_irq(struct pt_regs *regs) +{ + extern int i386_apic_get_critical_irq(struct pt_regs *regs); + return i386_apic_get_critical_irq(regs); +} +extern void CriticalInterruptException(struct pt_regs regs); +extern unsigned long critical_interrupts_map[]; +extern unsigned long is_privilege(struct pt_regs regs); +#endif /* _ASM_CRIT_INTR_H_ */ + diff -Naur linux-2.6.9/include/asm-i386/hardirq.h linux-2.6.9-lvlintr/include/asm-i386/hardirq.h --- linux-2.6.9/include/asm-i386/hardirq.h 2004-12-27 11:07:13.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/hardirq.h 2004-12-27 11:42:03.000000000 +0900 @@ -50,6 +50,13 @@ #define nmi_exit() (preempt_count() -= HARDIRQ_OFFSET) #define irq_enter() (preempt_count() += HARDIRQ_OFFSET) +#if defined(CONFIG_PRIORITY_INTR) +#define irq_exit_nobh() \ +do { \ + preempt_count() -= IRQ_EXIT_OFFSET; \ + preempt_enable_no_resched(); \ +} while (0) +#endif /* CONFIG_PRIORITY_INTR */ #define irq_exit() \ do { \ preempt_count() -= IRQ_EXIT_OFFSET; \ diff -Naur linux-2.6.9/include/asm-i386/io_apic.h linux-2.6.9-lvlintr/include/asm-i386/io_apic.h --- linux-2.6.9/include/asm-i386/io_apic.h 2004-12-27 11:07:13.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/io_apic.h 2004-12-27 11:42:03.000000000 +0900 @@ -52,7 +52,10 @@ #define ack_edge_ioapic ack_edge_ioapic_irq #define end_edge_ioapic end_edge_ioapic_irq #endif - +/* @@@@ for CT */ +extern void raise_irq(int irq); +extern void down_irq(int irq); +/* @@@@ end */ #define APIC_MISMATCH_DEBUG #define IO_APIC_BASE(idx) \ @@ -208,6 +211,4 @@ #define io_apic_assign_pci_irqs 0 #endif -extern int assign_irq_vector(int irq); - #endif diff -Naur linux-2.6.9/include/asm-i386/lvlintr.h linux-2.6.9-lvlintr/include/asm-i386/lvlintr.h --- linux-2.6.9/include/asm-i386/lvlintr.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/lvlintr.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,25 @@ +/* + * Prioritized Interruption Handling on i386 + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * Sun Zhitai + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#ifndef _ASM_LVLINTR_H_ +#define _ASM_LVLINTR_H_ +#include +#include +#include +#include +#include +#include +#include + +extern int set_hardware_mask(unsigned long current_level,unsigned long new_level); +extern unsigned long get_hardmask_level(void); +extern void check_hardmask_level(struct pt_regs regs); +#endif /* _ASM_LVLINTR_H_ */ + diff -Naur linux-2.6.9/include/asm-i386/lvlintr386.h linux-2.6.9-lvlintr/include/asm-i386/lvlintr386.h --- linux-2.6.9/include/asm-i386/lvlintr386.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/lvlintr386.h 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,188 @@ +/* + * Prioritized Interruption Handling on i386 ported from PPC. + * Author: Sun Zhitai(sun@cs.fujitsu.co.jp) + * + * 2004 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#ifndef __ASM_LVLINTR386_H__ +#define __ASM_LVLINTR386_H__ +#include +#include +#include +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) +#include +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ +#undef DEBUG_CHK_INVALID_SETHMSK +/* + * Hardware priority relevant definitions + */ +#define PRIVILEGED_IRQ_INTR_LEVEL (MAX_PRIVILEGED_LEVEL) + +extern int highest_pri_vec; +extern void __init resetup_vector(void); + +/* Total amount of mask level(zero(no mask level included) */ +#define NR_HWMSK 24 +#define MAX_PRIVILEGED_HWMSK 0xe9 +#define MIN_PRIVILEGED_HWMSK MAX_PRIVILEGED_HWMSK +#define MAX_PRIO_HWMSK highest_pri_vec +#define MIN_PRIO_HWMSK 0x31 +/* Hardware level mask information (zero level not included) */ +#define MAX_HWMSK 0xe9 +#define MIN_HWMSK 0x31 +/* Number of vectors for each cpu */ +#define PRIO_INTR_MAX_VECTOR_UP 256 +extern int handle_IRQ_event(unsigned int, struct pt_regs *, struct irqaction *); +#define HANDLE_IRQ_EVENT(irq, regs, action) handle_IRQ_event((unsigned int)(irq), (struct pt_regs *)(regs), (struct irqaction *)(action)) + +#define arch_setup_lvlintr() do{ resetup_vector(); } while (0) +struct sl_hmask_irq { + int soft_level; + int irq; + int hard_mask; + int vector; +}; + +extern struct sl_hmask_irq sl_hmask_irq[NR_IRQS]; +extern int hmask2smask[]; +extern int smask2hmask[]; + +/* + * Interrupt thread indexing method + */ +#define irq2index(irq) (irq) /* Does not support SMP */ + +/* + * Convert level to hardware mask priority + * if no hardware mask return 0. + */ +static __inline__ int +lvl2hwpri(int lvl) /* lvl is soft_level */ +{ + return smask2hmask[lvl]; +} +/* + * Convert hardware mask priority to level + * if no hardware mask return 0. + */ +static __inline__ int +hwpri2lvl(int pri) +{ + return hmask2smask[pri]; +} + +/* + * Is the IRQ under a hardware assist priority control? + */ +static __inline__ int +is_hard_masking_interrupt(int irq) +{ + if (sl_hmask_irq[irq].soft_level > MAX_NORM_LEVEL ) + return 1; + + return 0; +} +/* + * Start priority based interrupt handling + */ +static __inline__ void +start_lvlintr(void) +{ + unsigned long flags; + extern int reassign_flag; + __kern_intr_save(flags); + reassign_flag = 1; + switch_to_prio_intr(); + apic_write(APIC_TASKPRI, 0); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + if (register_tpri_trace_facility()) + printk (KERN_WARNING "Can not register TPR tracer\n"); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + __kern_intr_restore(flags); + return; +} +/* + * Set hardware level interrupt mask + */ +static __inline__ void +set_hard_lvl_mask(int irq, struct pt_regs *regs, struct irqaction *action) +{ + unsigned long mask; + extern unsigned long soft_mask_level; + extern void show_registers(struct pt_regs *regs); + extern void show_lvlmask_trace(void); + + if ( (likely(is_hard_masking_interrupt(irq))) && + (likely(sl_hmask_irq[irq].soft_level < MIN_PRIVILEGED_HWMSK)) && + (likely(!(action->flags & SA_INTERRUPT))) ) + { + __kern_intr_disable(); + if (!regs) { /* Normal interrupt case */ + apic_write(APIC_TASKPRI, 0); + soft_mask_level=0; + }else{ + mask = sl_hmask_irq[irq].hard_mask; +#if defined(DEBUG_CHK_INVALID_SETHMSK) + if ((regs->xes >> 16) %8 > 1 ) { + show_registers(regs); +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + show_lvlmask_trace(); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + printk("Restore check TASKPRI: 0x%lx\n",mask); + printk("XES:0x%x\n",regs->xes); + printk("XDS:0x%x\n",regs->xds); + printk("XCS:0x%x\n",regs->xcs); +while(1); + } +#endif /* DEBUG_CHK_INVALID_SETHMSK */ + + apic_write(APIC_TASKPRI, mask); + soft_mask_level=hmask2smask[mask]; + } + __kern_intr_enable(); /* local_irq_enable();*/ + } + return; +} +/* + * Unset hardware level interrupt mask + */ +static __inline__ void +restore_hard_lvl_mask(int irq, struct pt_regs *regs, struct irqaction *action) +{ + unsigned long mask; + extern unsigned long soft_mask_level; + extern void show_registers(struct pt_regs *regs); + extern void show_lvlmask_trace(void); + + __asm__ __volatile__("cli": : :"memory"); /* local_irq_disable(); */ + if (!regs) { /* Normal interrupt case */ + apic_write(APIC_TASKPRI, 0); + soft_mask_level=0; + }else{ + mask = regs->xes; + mask >>= 16; + mask &= APIC_TPRI_MASK; + soft_mask_level=hmask2smask[mask]; + apic_write(APIC_TASKPRI, mask); +#if defined(DEBUG_CHK_INVALID_SETHMSK) + if (mask%8 > 1) { +#if defined(CONFIG_PRIO_INTR_DIAG_TRACE_TPRI) + show_lvlmask_trace(); +#endif /* CONFIG_PRIO_INTR_DIAG_TRACE_TPRI */ + show_registers(regs); + printk("RestoreWrite TASKPRI: 0x%lx\n",mask); + printk("XES:0x%x\n",regs->xes); + printk("XDS:0x%x\n",regs->xds); + printk("XCS:0x%x\n",regs->xcs); + while(1); + } +#endif /* DEBUG_CHK_INVALID_SETHMSK */ + } + return; +} +#endif /* ASM_LVLINTR386_H__ */ + diff -Naur linux-2.6.9/include/asm-i386/mach-default/irq_vectors.h linux-2.6.9-lvlintr/include/asm-i386/mach-default/irq_vectors.h --- linux-2.6.9/include/asm-i386/mach-default/irq_vectors.h 2004-12-27 11:07:13.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/mach-default/irq_vectors.h 2004-12-27 14:23:46.000000000 +0900 @@ -55,6 +55,14 @@ * to work around the 'lost local interrupt if more than 2 IRQ * sources per level' errata. */ +/* + * The highest priority vector of Prioritized Interrupt processing. + */ + +#if defined(CONFIG_PRIORITY_INTR) +#define FIRST_PRIO_INTR_VECTOR 0xe9 +#endif /* defined(CONFIG_PRIORITY_PRIO) || defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) */ + #define LOCAL_TIMER_VECTOR 0xef /* diff -Naur linux-2.6.9/include/asm-i386/priointrdiag.h linux-2.6.9-lvlintr/include/asm-i386/priointrdiag.h --- linux-2.6.9/include/asm-i386/priointrdiag.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/priointrdiag.h 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,60 @@ +#if !defined(_X86_PRIO_INTRDIAG_H) +#define _X86_PRIO_INTRDIAG_H +#include +#include +#include + +#define get_lr(var) do{ \ + var=(unsigned long)__builtin_return_address(0); \ +}while(0) + +struct pt_regs; +struct task_struct; +/* + * Diagnostic operations + */ +typedef struct _prio_intr_diag_operations_t{ + void (*diag_assert)(int cond, const char *filename, unsigned long line, const char *function); + /* CPU level irq mask operations */ + void (*diag_local_irq_disable)(unsigned long ret_addr); + void (*diag_local_irq_enable)(unsigned long ret_addr); + void (*diag_local_irq_save)(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags); + void (*diag_local_irq_restore)(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags); + void (*diag_local_interrupt_save)(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags); + void (*diag_local_interrupt_restore)(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags); + /* interrupt mask operations */ + void (*diag_startup_irq)(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask,unsigned long new_irq_mask, unsigned long depth); + void (*diag_shutdown_irq)(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth); + void (*diag_enable_irq)(unsigned long ret_addr, unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth); + void (*diag_disable_irq)(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth); + void (*diag_enter_ack_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_exit_ack_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_enter_end_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_exit_end_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_enter_set_affinity_irq)(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth); + void (*diag_exit_set_affinity_irq)(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth); + /* Preemption relevant events */ + void (*diag_preempt_disable)(unsigned long ret_addr,unsigned long preempt_count); + void (*diag_preempt_enable)(unsigned long ret_addr,unsigned long preempt_count); + void (*diag_preempt_enable_no_resched)(unsigned long ret_addr,unsigned long preempt_count); + void (*diag_preempt_check_resched)(unsigned long ret_addr,unsigned long tif); + /* System Calls */ + void (*diag_syscall_enter)(struct pt_regs *regs); + void (*diag_syscall_exit)(struct pt_regs *regs); + void (*diag_irq_enter)(struct pt_regs *regs); + void (*diag_irq_exit)(struct pt_regs *regs); + void (*diag_restore)(struct pt_regs *regs); + /* Exceptions */ + void (*diag_page_fault)(struct pt_regs *, unsigned long, unsigned long,unsigned long ); + /* Schedule */ + void (*diag_schedule)(struct task_struct *prev,struct task_struct *next); +}prio_intr_diag_operations_t; + +/* + * interrupt operation which does not interfere diagnostic relevant functions + */ +#define _in_diag_irqs_disabled() irqs_disabled() +#define __in_diag_irq_save_flags(flags) __kern_intr_save(flags) +#define __in_diag_restore_flags(flags) __kern_intr_restore(flags) +#endif /* _X86_PRIO_INTRDIAG_H */ + diff -Naur linux-2.6.9/include/asm-i386/system.h linux-2.6.9-lvlintr/include/asm-i386/system.h --- linux-2.6.9/include/asm-i386/system.h 2004-12-27 11:07:13.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/system.h 2004-12-27 11:42:03.000000000 +0900 @@ -7,29 +7,13 @@ #include #include /* for LOCK_PREFIX */ +#ifdef CONFIG_PRIORITY_INTR +#include +#endif + #ifdef __KERNEL__ struct task_struct; /* one of the stranger aspects of C forward declarations.. */ -extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next)); - -#define switch_to(prev,next,last) do { \ - unsigned long esi,edi; \ - asm volatile("pushfl\n\t" \ - "pushl %%ebp\n\t" \ - "movl %%esp,%0\n\t" /* save ESP */ \ - "movl %5,%%esp\n\t" /* restore ESP */ \ - "movl $1f,%1\n\t" /* save EIP */ \ - "pushl %6\n\t" /* restore EIP */ \ - "jmp __switch_to\n" \ - "1:\t" \ - "popl %%ebp\n\t" \ - "popfl" \ - :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \ - "=a" (last),"=S" (esi),"=D" (edi) \ - :"m" (next->thread.esp),"m" (next->thread.eip), \ - "2" (prev), "d" (next)); \ -} while (0) - #define _set_base(addr,base) do { unsigned long __pr; \ __asm__ __volatile__ ("movw %%dx,%1\n\t" \ "rorl $16,%%edx\n\t" \ @@ -270,8 +254,8 @@ return old; } -#define cmpxchg(ptr,o,n)\ - ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ +#define cmpxchg(ptr,o,n) \ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o), \ (unsigned long)(n),sizeof(*(ptr)))) #ifdef __KERNEL__ @@ -442,6 +426,26 @@ /* interrupt control.. */ #define local_save_flags(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushfl ; popl %0":"=g" (x): /* no input */); } while (0) + +/* + Functions onto CONFIG_PRIORITY_INTR were ported from PPC's hw_irq.h + March 2004, by Sun Zhitai +*/ + +#if defined(CONFIG_PRIORITY_INTR) +#define local_irq_save(f) current_intr_mask_ops->local_irq_save_ptr((unsigned long *)(&(f))) +#define local_irq_restore(f) current_intr_mask_ops->local_irq_restore((unsigned long)(f)) +#define local_irq_disable() current_intr_mask_ops->local_irq_disable() +#define local_irq_enable() current_intr_mask_ops->local_irq_enable() +#define irqs_disabled() current_intr_mask_ops->irqs_disabled() +#define have_irqs_been_enabled(regs, f) current_intr_mask_ops->have_irqs_been_enabled((struct pt_regs *)(regs), (unsigned long)(f)) +#define local_interrupt_save(f) current_intr_mask_ops->local_interrupt_save_ptr((unsigned long *)(&(f))) +#define local_interrupt_restore(f) current_intr_mask_ops->local_interrupt_restore((unsigned long)(f)) +#define safe_halt() do { local_irq_enable(); __asm__ __volatile__("hlt": : :"memory");} while(0) + +#else /* !CONFIG_PRIORITY_INTR */ + + #define local_irq_restore(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc"); } while (0) #define local_irq_disable() __asm__ __volatile__("cli": : :"memory") #define local_irq_enable() __asm__ __volatile__("sti": : :"memory") @@ -457,10 +461,62 @@ /* For spinlocks etc */ #define local_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory") +#define local_interrupt_save(f) local_irq_save(f) +#define local_interrupt_restore(f) local_irq_restore(f) +#endif /* CONFIG_PRIORITY_INTR */ /* * disable hlt during certain critical i/o operations */ +#ifdef __KERNEL__ +extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next)); + +#if defined(CONFIG_PRIORITY_INTR) +#define switch_to(prev,next,last) do { \ + unsigned long esi,edi; \ + unsigned long flags; \ + local_interrupt_save(flags); \ + asm volatile("pushfl\n\t" \ + "pushl %%ebp\n\t" \ + "movl %%esp,%0\n\t" /* save ESP */ \ + "movl %5,%%esp\n\t" /* restore ESP */ \ + "movl $1f,%1\n\t" /* save EIP */ \ + "pushl %6\n\t" /* restore EIP */ \ + "jmp __switch_to\n" \ + "1:\t" \ + "popl %%ebp\n\t" \ + "popfl" \ + :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \ + "=a" (last),"=S" (esi),"=D" (edi) \ + :"m" (next->thread.esp),"m" (next->thread.eip), \ + "2" (prev), "d" (next)); \ + local_interrupt_restore(flags); \ +} while (0) + +#else /* !CONFIG_PRIORITY_INTR */ + +/* sleeping_thread_to_gdb_regs depends on this code. Correct it if you change + * any of the following */ +#define switch_to(prev,next,last) do { \ + unsigned long esi,edi; \ + asm volatile("pushfl\n\t" \ + "pushl %%ebp\n\t" \ + "movl %%esp,%0\n\t" /* save ESP */ \ + "movl %5,%%esp\n\t" /* restore ESP */ \ + "movl $1f,%1\n\t" /* save EIP */ \ + "pushl %6\n\t" /* restore EIP */ \ + "jmp __switch_to\n" \ + "1:\t" \ + "popl %%ebp\n\t" \ + "popfl" \ + :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \ + "=a" (last),"=S" (esi),"=D" (edi) \ + :"m" (next->thread.esp),"m" (next->thread.eip), \ + "2" (prev), "d" (next)); \ +} while (0) +#endif /* CONFIG_PRIORITY_INTR */ +#endif /* __KERNEL__ */ + #define HAVE_DISABLE_HLT void disable_hlt(void); void enable_hlt(void); diff -Naur linux-2.6.9/include/asm-i386/tpri_trace.h linux-2.6.9-lvlintr/include/asm-i386/tpri_trace.h --- linux-2.6.9/include/asm-i386/tpri_trace.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/tpri_trace.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,38 @@ +/* + * include/asm-i386/tpri_trace.h + * + * Author: Takeharu KATO (tkato@cs.fujitsu.co.jp) + * SUN Zhitai (sun@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#if !defined(_ASM_I386_TPRI_TRACE_H_) +#define _ASM_I386_TPRI_TRACE_H_ +#include +#if defined(CONFIG_TPRITRACE_MAX_TRACE) +#define MAX_TRACE CONFIG_TPRITRACE_MAX_TRACE +#else +#define MAX_TRACE 100 +#endif /* CONFIG_TPRITRACE_MAX_TRACE */ +#define TRACE_LOCAL_IRQ_DISABLE_NUM 1 +#define TRACE_LOCAL_IRQ_ENABLE_NUM 2 +#define TRACE_LOCAL_IRQ_SAVE_NUM 3 +#define TRACE_LOCAL_IRQ_RESTORE_NUM 4 +#define TRACE_LOCAL_INTERRUPT_SAVE_NUM 5 +#define TRACE_LOCAL_INTERRUPT_RESTORE_NUM 6 + +typedef struct __lvlmask_trace{ + unsigned long caller; + unsigned long op; + unsigned long tpr; + unsigned long arg1; + unsigned long arg2; +}lvlmask_trace_t; + +extern void show_lvlmask_trace(void); +extern int register_tpri_trace_facility(void); +extern void unregister_tpri_trace_facility(void); +#endif /* _ASM_I386_TPRI_TRACE_H_ */ diff -Naur linux-2.6.9/include/asm-i386/tpriops.h linux-2.6.9-lvlintr/include/asm-i386/tpriops.h --- linux-2.6.9/include/asm-i386/tpriops.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-i386/tpriops.h 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,37 @@ +/* + * include/asm-i386/tpriops.h + * + * Author: Takeharu KATO (tkato@cs.fujitsu.co.jp) + * SUN Zhitai (sun@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#if !defined(_ASM_I386_TPRIOPS_H_) +#define _ASM_I386_TPRIOPS_H_ +#include +#define __MSK_EFLAGS_SHIFT 24 +#define __PURE_EFLAGS_MSK 0xffffff +#define __TPRI_REG_MSK 0xff +#define __kern_intr_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory") +#define __kern_intr_restore(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc"); } while (0) +#define __kern_intr_disable() __asm__ __volatile__("cli": : :"memory") +#define __kern_intr_enable() __asm__ __volatile__("sti": : :"memory") + +typedef struct __prio_intr_mask_ops{ + unsigned char *name; + void (*local_interrupt_save_ptr)(unsigned long *f); + void (*local_interrupt_restore)(unsigned long f); + void (*local_irq_save_ptr)(unsigned long *f); + void (*local_irq_restore)(unsigned long f); + void (*local_irq_disable)(void); + void (*local_irq_enable)(void); + int (*irqs_disabled)(void); + int (*have_irqs_been_enabled)(struct pt_regs *regs, unsigned long f); +}prio_intr_mask_ops_t; +extern prio_intr_mask_ops_t *current_intr_mask_ops; +void switch_to_prio_intr(void); +void switch_to_linux(void); +#endif /* _ASM_I386_TPRIOPS_H_ */ diff -Naur linux-2.6.9/include/asm-ppc/bitops.h linux-2.6.9-lvlintr/include/asm-ppc/bitops.h --- linux-2.6.9/include/asm-ppc/bitops.h 2004-12-27 11:07:03.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-ppc/bitops.h 2004-12-27 11:42:03.000000000 +0900 @@ -276,17 +276,31 @@ * Find the first bit set in a 140-bit bitmap. * The first 100 bits are unlikely to be set. */ +#if defined(CONFIG_PRIORITY_INTR) +#include +#endif /* CONFIG_PRIORITY_INTR */ static inline int sched_find_first_bit(const unsigned long *b) { +#if defined(CONFIG_PRIORITY_INTR) + int i; +#endif /* CONFIG_PRIORITY_INTR */ if (unlikely(b[0])) return __ffs(b[0]); if (unlikely(b[1])) return __ffs(b[1]) + 32; if (unlikely(b[2])) return __ffs(b[2]) + 64; +#if defined(CONFIG_PRIORITY_INTR) + for(i=3;i<(MAX_PRIO>>5);++i) + if (b[i]) + return __ffs(b[i]) + (i<<5); + + return __ffs(b[(MAX_PRIO>>5)]) + ( (MAX_PRIO>>5)<<5); +#else if (b[3]) return __ffs(b[3]) + 96; return __ffs(b[4]) + 128; +#endif /* CONFIG_PRIORITY_INTR */ } /** diff -Naur linux-2.6.9/include/asm-ppc/crit_intr.h linux-2.6.9-lvlintr/include/asm-ppc/crit_intr.h --- linux-2.6.9/include/asm-ppc/crit_intr.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-ppc/crit_intr.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,27 @@ +/* + * PowerPC Critical Interrupt + * Author: Takeharu KATO (tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ + +#ifndef _ASM_PPC_CRIT_INTR_H_ +#define _ASM_PPC_CRIT_INTR_H_ +#include +extern __inline__ int +get_critical_irq(struct pt_regs *regs) +{ +#if defined(CONFIG_4xx) + extern int ppc405_pic_get_critical_irq(struct pt_regs *regs); + return ppc405_pic_get_critical_irq(regs); +#else + return -1; +#endif +} +extern void CriticalInterruptException(struct pt_regs *regs); +extern unsigned long critical_interrupts_map[]; +extern void set_irq_critical(int irq); +#endif /* _ASM_PPC_CRIT_INTR_H_ */ diff -Naur linux-2.6.9/include/asm-ppc/hw_irq.h linux-2.6.9-lvlintr/include/asm-ppc/hw_irq.h --- linux-2.6.9/include/asm-ppc/hw_irq.h 2004-12-27 11:07:04.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-ppc/hw_irq.h 2004-12-27 14:23:46.000000000 +0900 @@ -7,6 +7,9 @@ #include #include +#if defined(CONFIG_PRIO_INTR_DIAG) +#include +#endif /* CONFIG_PRIO_INTR_DIAG */ extern void timer_interrupt(struct pt_regs *); extern void ppc_irq_dispatch_handler(struct pt_regs *regs, int irq); @@ -15,38 +18,83 @@ #define irqs_disabled() ((mfmsr() & MSR_EE) == 0) +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +static inline void local_interrupt_save_ptr(unsigned long *flags) +{ + unsigned long msr; + msr = mfmsr(); + *flags = msr; + + mtmsr(msr & ~(MSR_EE|MSR_CE)); + __asm__ __volatile__("": : :"memory"); +#if defined(CONFIG_PRIO_INTR_DIAG) + prio_intr_local_interrupt_save_wrapper(msr,(msr & ~(MSR_EE|MSR_CE))); +#endif /* CONFIG_PRIO_INTR_DIAG */ +} +#define local_interrupt_save(flags) local_interrupt_save_ptr(&flags) +#if defined(CONFIG_PRIO_INTR_DIAG) +#define local_interrupt_restore(flags) do{ \ + prio_intr_local_interrupt_restore_wrapper(mfmsr(),flags); \ + mtmsr(flags); \ +}while(0) +#else +#define local_interrupt_restore(flags) mtmsr(flags) +#endif /* CONFIG_PRIO_INTR_DIAG */ +#else +#define local_interrupt_save(flags) local_irq_save(flags) +#define local_interrupt_restore(flags) local_irq_restore(flags) +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ #ifdef INLINE_IRQS +static inline void local_irq_save_ptr(unsigned long *flags) +{ + unsigned long msr; + + msr = mfmsr(); + *flags = msr; + + mtmsr(msr & ~MSR_EE); + __asm__ __volatile__("": : :"memory"); +#if defined(CONFIG_PRIO_INTR_DIAG) + prio_intr_local_irq_save_wrapper(msr,(msr & ~MSR_EE)); +#endif /* CONFIG_PRIO_INTR_DIAG */ +} static inline void local_irq_disable(void) { unsigned long msr; msr = mfmsr(); mtmsr(msr & ~MSR_EE); __asm__ __volatile__("": : :"memory"); + +#if defined(CONFIG_PRIO_INTR_DIAG) + prio_intr_local_irq_disable_wrapper(); +#endif /* CONFIG_PRIO_INTR_DIAG */ } static inline void local_irq_enable(void) { unsigned long msr; +#if defined(CONFIG_PRIO_INTR_DIAG) + prio_intr_local_irq_enable_wrapper(); +#endif /* CONFIG_PRIO_INTR_DIAG */ + + __asm__ __volatile__("": : :"memory"); msr = mfmsr(); mtmsr(msr | MSR_EE); } -static inline void local_irq_save_ptr(unsigned long *flags) -{ - unsigned long msr; - msr = mfmsr(); - *flags = msr; - mtmsr(msr & ~MSR_EE); - __asm__ __volatile__("": : :"memory"); -} - #define local_save_flags(flags) ((flags) = mfmsr()) #define local_irq_save(flags) local_irq_save_ptr(&flags) -#define local_irq_restore(flags) mtmsr(flags) - +#if defined(CONFIG_PRIO_INTR_DIAG) +#define local_irq_restore(flags) do{ \ + prio_intr_local_irq_restore_wrapper(mfmsr(),flags); \ + mtmsr(flags); \ +}while(0) #else +#define local_irq_restore(flags) mtmsr(flags) +#endif /* CONFIG_PRIO_INTR_DIAG */ +#else /* !INLINE_IRQS */ extern void local_irq_enable(void); extern void local_irq_disable(void); @@ -56,8 +104,7 @@ #define local_save_flags(flags) local_save_flags_ptr(&flags) #define local_irq_save(flags) ({local_save_flags(flags);local_irq_disable();}) -#endif - +#endif /* INLINE_IRQS */ extern void do_lost_interrupts(unsigned long); #define mask_irq(irq) ({if (irq_desc[irq].handler && irq_desc[irq].handler->disable) irq_desc[irq].handler->disable(irq);}) diff -Naur linux-2.6.9/include/asm-ppc/irq.h linux-2.6.9-lvlintr/include/asm-ppc/irq.h --- linux-2.6.9/include/asm-ppc/irq.h 2004-12-27 11:07:03.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-ppc/irq.h 2004-12-27 11:42:03.000000000 +0900 @@ -321,6 +321,10 @@ extern unsigned long ppc_cached_irq_mask[NR_MASK_WORDS]; extern unsigned long ppc_lost_interrupts[NR_MASK_WORDS]; extern atomic_t ppc_n_lost_interrupts; +#if defined(CONFIG_PRIORITY_INTR) +struct irqaction; +extern void handle_irq_event(int, struct pt_regs *, struct irqaction *); +#endif /* CONFIG_PRIORITY_INTR */ struct irqaction; struct pt_regs; diff -Naur linux-2.6.9/include/asm-ppc/lvlintr.h linux-2.6.9-lvlintr/include/asm-ppc/lvlintr.h --- linux-2.6.9/include/asm-ppc/lvlintr.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-ppc/lvlintr.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,33 @@ +/* + * PowerPC Priority based Interruption Handling + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#ifndef _ASM_PPC_LVLINTR_H_ +#define _ASM_PPC_LVLINTR_H_ +#if defined(CONFIG_4xx) +#include +#include +#include +#include +#include + +/* + * Chip select among IBM 4xx series + */ +#if defined(CONFIG_4xx) +#include +#else +#error "Current version of priority based interrupt does not support this archtecture" +#endif /* Chip select(CONFIG_4xx) */ +#else +#error "Current version of priority based interrupt does not support this archtecture" +#endif /* CONFIG_4x */ +extern int set_hardware_mask(unsigned long current_level,unsigned long new_level); +extern unsigned long get_hardmask_level(void); +#endif /* _ASM_PPC_LVLINTR_H_ */ diff -Naur linux-2.6.9/include/asm-ppc/priointrdiag.h linux-2.6.9-lvlintr/include/asm-ppc/priointrdiag.h --- linux-2.6.9/include/asm-ppc/priointrdiag.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/asm-ppc/priointrdiag.h 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,79 @@ +#if !defined(_PPC_PRIO_INTRDIAG_H) +#define _PPC_PRIO_INTRDIAG_H +#include +#include +#define get_lr(var) do{ \ + __asm__ __volatile__ ("mflr %0\n\t":"=r"(var)); \ +}while(0) + +struct pt_regs; +struct task_struct; +/* + * Diagnostic operations + */ +typedef struct _prio_intr_diag_operations_t{ + void (*diag_assert)(int cond, const char *filename, unsigned long line, const char *function); + /* CPU level irq mask operations */ + void (*diag_local_irq_disable)(unsigned long ret_addr); + void (*diag_local_irq_enable)(unsigned long ret_addr); + void (*diag_local_irq_save)(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags); + void (*diag_local_irq_restore)(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags); + void (*diag_local_interrupt_save)(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags); + void (*diag_local_interrupt_restore)(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags); + /* interrupt mask operations */ + void (*diag_startup_irq)(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask,unsigned long new_irq_mask, unsigned long depth); + void (*diag_shutdown_irq)(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth); + void (*diag_enable_irq)(unsigned long ret_addr, unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth); + void (*diag_disable_irq)(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth); + void (*diag_enter_ack_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_exit_ack_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_enter_end_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_exit_end_irq)(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth); + void (*diag_enter_set_affinity_irq)(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth); + void (*diag_exit_set_affinity_irq)(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth); + /* Preemption relevant events */ + void (*diag_preempt_disable)(unsigned long ret_addr,unsigned long preempt_count); + void (*diag_preempt_enable)(unsigned long ret_addr,unsigned long preempt_count); + void (*diag_preempt_enable_no_resched)(unsigned long ret_addr,unsigned long preempt_count); + void (*diag_preempt_check_resched)(unsigned long ret_addr,unsigned long tif); + /* System Calls */ + void (*diag_syscall_enter)(struct pt_regs *regs); + void (*diag_syscall_exit)(struct pt_regs *regs); + void (*diag_irq_enter)(struct pt_regs *regs); + void (*diag_irq_exit)(struct pt_regs *regs); + void (*diag_restore)(struct pt_regs *regs); + /* Exceptions */ + void (*diag_page_fault)(struct pt_regs *, unsigned long, unsigned long,unsigned long ); + /* Schedule */ + void (*diag_schedule)(struct task_struct *prev,struct task_struct *next); +}prio_intr_diag_operations_t; + +/* + * interrupt operation which does not interfere diagnostic relevant functions + */ +#define _in_diag_mfmsr() ({unsigned int rval; \ + asm volatile("mfmsr %0" : "=r" (rval)); rval;}) +#define _in_diag_mtmsr(v) asm volatile("mtmsr %0" : : "r" (v)) + +#define _in_diag_irqs_disabled() ((mfmsr() & MSR_EE) == 0) + +static inline void +__ppc_in_diag_irq_save_flags(unsigned long *flags) +{ + unsigned long msr; + + msr = _in_diag_mfmsr(); + *flags = msr; + + _in_diag_mtmsr(msr & ~MSR_EE); + __asm__ __volatile__("": : :"memory"); +} +static inline void +__ppc_in_diag_restore_flags(unsigned long *flags) +{ + _in_diag_mtmsr(*flags); + __asm__ __volatile__("": : :"memory"); +} +#define __in_diag_irq_save_flags(flags) __ppc_in_diag_irq_save_flags(&flags) +#define __in_diag_restore_flags(flags) __ppc_in_diag_restore_flags(&flags) +#endif /* _PPC_PRIO_INTRDIAG_H */ diff -Naur linux-2.6.9/include/linux/interrupt.h linux-2.6.9-lvlintr/include/linux/interrupt.h --- linux-2.6.9/include/linux/interrupt.h 2004-12-27 11:07:05.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/interrupt.h 2004-12-27 11:42:03.000000000 +0900 @@ -246,5 +246,7 @@ extern unsigned long probe_irq_on(void); /* returns 0 on failure */ extern int probe_irq_off(unsigned long); /* returns 0 or negative on failure */ extern unsigned int probe_irq_mask(unsigned long); /* returns mask of ISA interrupts */ - +#if defined(CONFIG_PRIORITY_INTR_NORMAL) +void wakeup_softirqd(void); +#endif /* CONFIG_PRIORITY_NORMAL */ #endif diff -Naur linux-2.6.9/include/linux/lvlintr.h linux-2.6.9-lvlintr/include/linux/lvlintr.h --- linux-2.6.9/include/linux/lvlintr.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/lvlintr.h 2004-12-28 08:54:46.000000000 +0900 @@ -0,0 +1,81 @@ +/* + * Priority based Interruption Handling + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#ifndef _LINUX_LVLINTR_H_ +#define _LINUX_LVLINTR_H_ +#include +#if defined(CONFIG_PRIORITY_INTR) +#include +#include +#include +#include +#include +#include + +#define PRIO_INTR_BANNER "Priority based interrupt enabled" + +#define PRIO_INTR_WAIT 0 +#define PRIO_INTR_ACTIVE 1 +#define PRIO_INTR_PENDING 2 + +#define spin_lock_interruptsave(lock, flags) \ +do { \ + local_interrupt_save(flags); \ + preempt_disable(); \ + _raw_spin_lock(lock); \ +} while (0) + +#define spin_unlock_interruptrestore(lock, flags) \ +do { \ + _raw_spin_unlock(lock); \ + local_interrupt_restore(flags); \ + preempt_enable(); \ +} while (0) + +typedef enum +{ + NORMAL_INTR=0, + PRIO_INTR, + PRIVILEGED_INTR +}interrupt_type_t; + +typedef struct _interrupt_thread_info{ + struct task_struct *irq_thread; + int state; +}interrupt_thread_info_t; + +/* + * get inerrupt type + */ +static __inline__ interrupt_type_t +get_intr_type(int lvl) +{ + if ( (MIN_PRIVILEGED_LEVEL <= lvl) && + (MAX_PRIVILEGED_LEVEL >= lvl) ) + return PRIVILEGED_INTR; + + if ( (MIN_PRIO_LEVEL <= lvl) && + (MAX_PRIO_LEVEL >= lvl) ) + return PRIO_INTR; + + return NORMAL_INTR; +} + + +extern unsigned long current_level; +int wakeup_interrupt_thread(int irq); +int convert_irq2level(int irq); +int set_interrupt_mask(unsigned long level,unsigned long *old_level); +/* + * Plat form dependent definitions + */ +#include +#endif /* CONFIG_PRIORITY_INTR */ +#endif /* _LINUX_LVLINTR_H_ */ diff -Naur linux-2.6.9/include/linux/preempt.h linux-2.6.9-lvlintr/include/linux/preempt.h --- linux-2.6.9/include/linux/preempt.h 2004-12-27 11:07:05.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/preempt.h 2004-12-27 14:23:46.000000000 +0900 @@ -8,6 +8,7 @@ #include #include +#include #define preempt_count() (current_thread_info()->preempt_count) @@ -23,11 +24,12 @@ #ifdef CONFIG_PREEMPT -asmlinkage void preempt_schedule(void); - +extern void preempt_schedule(void); +#if defined(CONFIG_PRIO_INTR_DIAG) #define preempt_disable() \ do { \ inc_preempt_count(); \ + prio_intr_preempt_disable_wrapper(preempt_count()); \ barrier(); \ } while (0) @@ -35,21 +37,67 @@ do { \ barrier(); \ dec_preempt_count(); \ + prio_intr_preempt_enable_no_resched_wrapper(preempt_count()); \ } while (0) - +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +extern void do_privileged_irq_requests(void); #define preempt_check_resched() \ do { \ + prio_intr_preempt_check_resched_wrapper(current_thread_info()->flags); \ + do_privileged_irq_requests(); \ + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + preempt_schedule(); \ +} while (0) +#else +#define preempt_check_resched() \ +do { \ + prio_intr_preempt_check_resched_wrapper(current_thread_info()->flags); \ if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ preempt_schedule(); \ } while (0) +#endif #define preempt_enable() \ do { \ preempt_enable_no_resched(); \ + prio_intr_preempt_enable_wrapper(preempt_count()); \ preempt_check_resched(); \ } while (0) #else +#define preempt_disable() \ +do { \ + inc_preempt_count(); \ + barrier(); \ +} while (0) + +#define preempt_enable_no_resched() \ +do { \ + dec_preempt_count(); \ + barrier(); \ +} while (0) +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +extern void do_privileged_irq_requests(void); +#define preempt_check_resched() \ +do { \ + do_privileged_irq_requests(); \ + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + preempt_schedule(); \ +} while (0) +#else +#define preempt_check_resched() \ +do { \ + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + preempt_schedule(); \ +} while (0) +#endif +#define preempt_enable() \ +do { \ + preempt_enable_no_resched(); \ + preempt_check_resched(); \ +} while (0) +#endif /* CONFIG_PRIO_INTR_DIAG */ +#else #define preempt_disable() do { } while (0) #define preempt_enable_no_resched() do { } while (0) diff -Naur linux-2.6.9/include/linux/priointrdiag.h linux-2.6.9-lvlintr/include/linux/priointrdiag.h --- linux-2.6.9/include/linux/priointrdiag.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/priointrdiag.h 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,215 @@ +/* + * - Diagnostic facilities - + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#if !defined(_LINUX_PRIO_INTRDIAG_H) +#define _LINUX_PRIO_INTRDIAG_H +#include +#include +#if defined(CONFIG_PRIO_INTR_DIAG) +#include +#include +extern prio_intr_diag_operations_t *prio_intr_diag_ops; +extern void prio_intr_local_interrupt_save_wrapper(unsigned long,unsigned long); +extern void prio_intr_local_interrupt_restore_wrapper(unsigned long,unsigned long); +extern void prio_intr_local_irq_save_wrapper(unsigned long, unsigned long); +extern void prio_intr_local_irq_restore_wrapper(unsigned long , unsigned long); +extern void prio_intr_local_irq_enable_wrapper(void); +extern void prio_intr_local_irq_disable_wrapper(void); +extern void prio_intr_preempt_disable_wrapper(unsigned long); +extern void prio_intr_preempt_enable_no_resched_wrapper(unsigned long); +extern void prio_intr_preempt_check_resched_wrapper(unsigned long); +extern void prio_intr_preempt_enable_wrapper(unsigned long); + +/* + * Debug Flags + */ +#define PRIO_INTR_DIAG_DEBUG + +/* + * Wrapper macro/functions + */ +#if defined(PRIO_INTR_DIAG_DEBUG) +#define prio_intr_invoke_diag_func(name,arg...) do{ \ + if ( (prio_intr_diag_ops) && (prio_intr_diag_ops->name) ) { \ + prio_intr_diag_ops->name(arg); \ + }else{ \ + printk(KERN_ALERT "Undefined function " #name "\n"); \ + } \ +}while(0) +#else +#define prio_intr_invoke_diag_func(name,arg...) do{ \ + prio_intr_diag_ops->name(arg); \ +}while(0) +#endif /* PRIO_INTR_DIAG_DEBUG */ +#define prio_intr_diag_assert(cond) do{ \ + if (!(cond)) \ + prio_intr_invoke_diag_func(diag_assert,cond, __FILE__, __LINE__, __FUNCTION__); \ +}while(0) +static __inline__ void +prio_intr_diag_local_irq_disable(unsigned long ret_addr) +{ + prio_intr_invoke_diag_func(diag_local_irq_disable,ret_addr); +} +static __inline__ void +prio_intr_diag_local_irq_enable(unsigned long ret_addr) +{ + prio_intr_invoke_diag_func(diag_local_irq_enable,ret_addr); +} +static __inline__ void +prio_intr_diag_local_irq_save(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ + prio_intr_invoke_diag_func(diag_local_irq_save,ret_addr,oldflags,newflags); +} +static __inline__ void +prio_intr_diag_local_irq_restore(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ + prio_intr_invoke_diag_func(diag_local_irq_restore,ret_addr,oldflags,newflags); +} +static __inline__ void +prio_intr_diag_local_interrupt_save(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags) +{ + prio_intr_invoke_diag_func(diag_local_interrupt_save,ret_addr,oldflags,newflags); +} +static __inline__ void +prio_intr_diag_local_interrupt_restore(unsigned long ret_addr, unsigned long oldflags,unsigned long newflags) +{ + prio_intr_invoke_diag_func(diag_local_interrupt_restore,ret_addr,oldflags,newflags); +} +/* interrupt mask operations */ +static __inline__ void +prio_intr_diag_startup_irq(unsigned long ret_addr,unsigned int irq, unsigned long omask, unsigned long nmask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_startup_irq,ret_addr,irq,omask,nmask,depth); +} +static __inline__ void +prio_intr_diag_shutdown_irq(unsigned long ret_addr,unsigned int irq, unsigned long omask, unsigned long nmask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_shutdown_irq,ret_addr,irq,omask,nmask,depth); +} +static __inline__ void +prio_intr_diag_enable_irq(unsigned long ret_addr,unsigned int irq,unsigned long omask, unsigned long nmask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_enable_irq,ret_addr,irq,omask,nmask,depth); +} +static __inline__ void +prio_intr_diag_disable_irq(unsigned long ret_addr,unsigned int irq,unsigned long omask, unsigned long nmask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_disable_irq,ret_addr,irq,omask,nmask,depth); +} +static __inline__ void +prio_intr_diag_enter_ack_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_enter_ack_irq,ret_addr,irq,irq_mask,depth); +} +static __inline__ void +prio_intr_diag_exit_ack_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_exit_ack_irq,ret_addr,irq,irq_mask,depth); +} +static __inline__ void +prio_intr_diag_enter_end_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_enter_end_irq,ret_addr,irq,irq_mask,depth); +} +static __inline__ void +prio_intr_diag_exit_end_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_exit_end_irq,ret_addr,irq,irq_mask,depth); +} +static __inline__ void +prio_intr_diag_enter_set_affinity_irq(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_enter_set_affinity_irq,ret_addr,irq,mask,irq_mask,depth); +} +static __inline__ void +prio_intr_diag_exit_set_affinity_irq(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth) +{ + prio_intr_invoke_diag_func(diag_exit_set_affinity_irq,ret_addr,irq,mask,irq_mask,depth); +} +static __inline__ void +prio_intr_diag_preempt_disable(unsigned long ret_addr,unsigned long preempt_count) +{ + prio_intr_invoke_diag_func(diag_preempt_disable,ret_addr,preempt_count); +} +static __inline__ void +prio_intr_diag_preempt_enable(unsigned long ret_addr,unsigned long preempt_count) +{ + prio_intr_invoke_diag_func(diag_preempt_enable,ret_addr,preempt_count); +} +static __inline__ void +prio_intr_diag_preempt_enable_no_resched(unsigned long ret_addr,unsigned long preempt_count) +{ + prio_intr_invoke_diag_func(diag_preempt_enable_no_resched,ret_addr,preempt_count); +} +static __inline__ void +prio_intr_diag_preempt_check_resched(unsigned long ret_addr,unsigned long tif) +{ + prio_intr_invoke_diag_func(diag_preempt_check_resched,ret_addr,tif); +} +static __inline__ void +prio_intr_diag_irq_enter(struct pt_regs *regs) +{ + prio_intr_invoke_diag_func(diag_irq_enter,regs); +} +static __inline__ void +prio_intr_diag_irq_exit(struct pt_regs *regs) +{ + prio_intr_invoke_diag_func(diag_irq_exit,regs); +} +static __inline__ void +prio_intr_diag_page_fault(struct pt_regs *regs, unsigned long address, unsigned long ecode,unsigned long type) +{ + prio_intr_invoke_diag_func(diag_page_fault,regs,address,ecode,type); +} +static __inline__ void +prio_intr_diag_schedule(struct task_struct *prev, struct task_struct *next) +{ + prio_intr_invoke_diag_func(diag_schedule,prev,next); +} + +extern int register_prio_intr_diag_operation(prio_intr_diag_operations_t *diag_ops); +extern int unregister_prio_intr_diag_operation(prio_intr_diag_operations_t *diag_ops); +extern void prio_intr_diag_syscall_enter(struct pt_regs *regs); +extern void prio_intr_diag_syscall_exit(struct pt_regs *regs); +extern void prio_intr_diag_restore(struct pt_regs *regs); +#else +#define prio_intr_diag_assert(cond) do{}while(0) +#define prio_intr_diag_local_irq_disable(ret_addr) do{}while(0) +#define prio_intr_diag_local_irq_enable(ret_addr) do{}while(0) +#define prio_intr_diag_local_irq_save(raddr,oflags,nflags) do{}while(0) +#define prio_intr_diag_local_irq_restore(raddr,oflags,nflags) do{}while(0) +#define prio_intr_diag_local_interrupt_save(raddr,oflags,nflags)do{}while(0) +#define prio_intr_diag_local_interrupt_restore(raddr,oflags,nflags) do{}while(0) +/* interrupt mask operations */ +#define prio_intr_diag_startup_irq(ret_addr,irq, omask, nmask, depth) do{}while(0) +#define prio_intr_diag_shutdown_irq(ret_addr,irq, omask, nmask, depth) do{}while(0) +#define prio_intr_diag_enable_irq(ret_addr,irq, omask, nmask,depth) do{}while(0) +#define prio_intr_diag_disable_irq(ret_addr,irq, omask,nmask,depth) do{}while(0) +#define prio_intr_diag_enter_ack_irq(ret_addr,irq, irq_mask, depth) do{}while(0) +#define prio_intr_diag_exit_ack_irq(ret_addr,irq, irq_mask, depth) do{}while(0) +#define prio_intr_diag_enter_end_irq(ret_addr,irq, irq_mask, depth) do{}while(0) +#define prio_intr_diag_exit_end_irq(ret_addr,irq, irq_mask, depth) do{}while(0) +#define prio_intr_diag_enter_set_affinity_irq(irq,mask, irq_mask, depth) do{}while(0) +#define prio_intr_diag_exit_set_affinity_irq(irq, mask, irq_mask, depth) do{}while(0) +#define prio_intr_diag_preempt_disable(ret_addr,preempt_count) do{}while(0) +#define prio_intr_diag_preempt_enable(ret_addr,preempt_count) do{}while(0) +#define prio_intr_diag_preempt_enable_no_resched(ret_addr,preempt_count) do{}while(0) +#define prio_intr_diag_preempt_check_resched(ret_addr,tif) do{}while(0) +#define prio_intr_diag_syscall_enter(regs) do{}while(0) +#define prio_intr_diag_syscall_exit(regs) do{}while(0) +#define prio_intr_diag_irq_enter(regs) do{}while(0) +#define prio_intr_diag_irq_exit(regs) do{}while(0) +#define prio_intr_diag_restore(regs) do{}while(0) +/* Exceptions */ +#define prio_intr_diag_page_fault(regs,address,ecode,type) do{}while(0) +/* Schedule */ +#define prio_intr_diag_schedule(prev,next) do{}while(0) +#endif /* CONFIG_PRIO_INTR_DIAG */ +#endif /* _LINUX_PRIO_INTRDIAG_H */ diff -Naur linux-2.6.9/include/linux/priv_irq.h linux-2.6.9-lvlintr/include/linux/priv_irq.h --- linux-2.6.9/include/linux/priv_irq.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/priv_irq.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,48 @@ +/* + * Prioritized Interruption Handling + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#ifndef _LINUX_QUICK_IRQ_H_ +#define _LINUX_QUICK_IRQ_H_ +#include +#include +#include +#include +#include +typedef struct _exp_sig{ + pid_t pid; + int signo; +}exp_sig_t; +typedef struct _privileged_irq_events{ + struct list_head list; + void (*func)(void *arg); + void *arg; + unsigned long count; +}privileged_irq_events_t; +#if defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) +#define PRIO_PRIVILEGED_BANNER "Privileged irq facility is enabled." +extern unsigned long enqueue_privileged_irq_request(privileged_irq_events_t *event); +extern int dequeue_privileged_irq_request(privileged_irq_events_t *event); +extern void wake_up_interruptible_sync_from_privileged_irq(wait_queue_head_t *q,privileged_irq_events_t *event); +extern void send_signal_from_privileged_irq(exp_sig_t *sig, privileged_irq_events_t *event); +#else +#define enqueue_privileged_irq_request(event) (0) +#define dequeue_privileged_irq_request(event) do{}while(0) +extern __inline__ void +wake_up_interruptible_sync_from_privileged_irq(wait_queue_head_t *q,privileged_irq_events_t *event) +{ + wake_up_interruptible_sync(q); +} +extern __inline__ void send_signal_from_privileged_irq(exp_sig_t *sig, privileged_irq_events_t *event) +{ + kill_proc(sig->pid, sig->signo, 0); +} +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ +#endif /* _LINUX_QUICK_IRQ_H_ */ + diff -Naur linux-2.6.9/include/linux/sched.h linux-2.6.9-lvlintr/include/linux/sched.h --- linux-2.6.9/include/linux/sched.h 2004-12-27 11:07:05.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/sched.h 2004-12-27 12:56:32.000000000 +0900 @@ -54,6 +54,7 @@ #define CLONE_CHILD_SETTID 0x01000000 /* set the TID in the child */ #define CLONE_STOPPED 0x02000000 /* Start in stopped state */ + /* * List of flags we want to share for kernel threads, * if only because they are not used by them anyway. @@ -326,14 +327,18 @@ * priority to a value higher than any user task. Note: * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO. */ - +#if defined(CONFIG_PRIORITY_INTR) +/* The scheduler relevant parameters defined linux/sched_param.h + * for priority based interrupt handling facility. + */ +#include +#else #define MAX_USER_RT_PRIO 100 #define MAX_RT_PRIO MAX_USER_RT_PRIO #define MAX_PRIO (MAX_RT_PRIO + 40) - +#endif /* CONFIG_PRIORITY_INTR */ #define rt_task(p) (unlikely((p)->prio < MAX_RT_PRIO)) - /* * Some day this will be a full-fledged user tracking system.. */ @@ -640,6 +645,10 @@ #define sched_exec() {} #endif +#if defined(CONFIG_PRIORITY_INTR) +extern int setscheduler(pid_t pid, int policy, struct sched_param __user *param); +#endif /* CONFIG_PRIORITY_INTR */ + extern void sched_idle_next(void); extern void set_user_nice(task_t *p, long nice); extern int task_prio(const task_t *p); diff -Naur linux-2.6.9/include/linux/sched_param.h linux-2.6.9-lvlintr/include/linux/sched_param.h --- linux-2.6.9/include/linux/sched_param.h 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/include/linux/sched_param.h 2004-12-27 11:42:03.000000000 +0900 @@ -0,0 +1,42 @@ +/* + * Priority based Interruption Handling relevant sceduler definitions + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + * + */ +#ifndef _LINUX_SCHED_PARAM_H +#define _LINUX_SCHED_PARAM_H +/* + * 512 Privileged interrupts + * 385 -- 511 Reserved for privileged interrupts + * 257 -- 384 Priority interrupts(included reserved values.) + * 129 -- 256 Normal interrupts + * 100 -- 128 Reserved + * 1 -- 99 RT user processes + */ +#define LVL_INTERVAL 128 +#define __next_lvl(premax) ( (premax) - (LVL_INTERVAL) ) +#define MAX_INTR_LEVEL 512 +#define MAX_PRIVILEGED_LEVEL (MAX_INTR_LEVEL) +#define MIN_PRIVILEGED_LEVEL (__next_lvl(MAX_PRIVILEGED_LEVEL)+1) +#define MAX_PRIO_LEVEL (__next_lvl(MAX_PRIVILEGED_LEVEL)) +#define MIN_PRIO_LEVEL (__next_lvl(MAX_PRIO_LEVEL)+1) +#define MAX_NORM_LEVEL (__next_lvl(MAX_PRIO_LEVEL)) +#define MIN_NORM_LEVEL (__next_lvl(MAX_NORM_LEVEL)+1) +/* + * MAX_PRIO need to be defined here. + * It's used in asm/bitops.h. + */ +#ifndef MAX_PRIO +#define MAX_LVLD_RT_PRIO ((MAX_NORM_LEVEL)+1) +#define MAX_USER_RT_PRIO MAX_LVLD_RT_PRIO +#define MAX_RT_PRIO MAX_LVLD_RT_PRIO + +#define MAX_PRIO (MAX_RT_PRIO + 40) +#endif /* !MAX_PRIO */ + +#endif /* _LINUX_SCHED_PARAM_H */ diff -Naur linux-2.6.9/kernel/Makefile linux-2.6.9-lvlintr/kernel/Makefile --- linux-2.6.9/kernel/Makefile 2004-12-27 11:08:07.000000000 +0900 +++ linux-2.6.9-lvlintr/kernel/Makefile 2004-12-27 14:23:46.000000000 +0900 @@ -9,7 +9,7 @@ rcupdate.o intermodule.o extable.o params.o posix-timers.o \ kthread.o -obj-$(CONFIG_FUTEX) += futex.o +obj-$(CONFIG_FUTEX) += futex.o obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o obj-$(CONFIG_SMP) += cpu.o spinlock.o obj-$(CONFIG_UID16) += uid16.o @@ -24,6 +24,9 @@ obj-$(CONFIG_AUDIT) += audit.o obj-$(CONFIG_AUDITSYSCALL) += auditsc.o obj-$(CONFIG_KPROBES) += kprobes.o +obj-$(CONFIG_PRIORITY_INTR) += lvlintr.o +obj-$(CONFIG_PRIORITY_PRIVILEGED_IRQ) += priv_irq.o +obj-$(CONFIG_PRIO_INTR_DIAG) += priointrdiag.o ifneq ($(CONFIG_IA64),y) # According to Alan Modra , the -fno-omit-frame-pointer is diff -Naur linux-2.6.9/kernel/elinuxdiag.c linux-2.6.9-lvlintr/kernel/elinuxdiag.c --- linux-2.6.9/kernel/elinuxdiag.c 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/kernel/elinuxdiag.c 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,336 @@ +/* + * Priority based interrupt diagnostic facility. + * Author: Takeharu KATO(tkato@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#include +#include +#include +#include +#include + +static void +prio_intr_default_diag_assert(int cond, const char *filename, unsigned long line, const char *function){ +} +static void +prio_intr_default_diag_local_irq_disable(unsigned long ret_addr){ + +} +static void +prio_intr_default_diag_local_irq_enable(unsigned long ret_addr) +{ +} +static void +prio_intr_default_diag_local_irq_save(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ +} +static void +prio_intr_default_diag_local_irq_restore(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ +} +static void +prio_intr_default_diag_local_interrupt_save(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ +} +static void +prio_intr_default_diag_local_interrupt_restore(unsigned long ret_addr, unsigned long oldflags, unsigned long newflags) +{ +} + /* interrupt mask operations */ +static void +prio_intr_default_diag_startup_irq(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask,unsigned long new_irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_shutdown_irq(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_enable_irq(unsigned long ret_addr, unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_disable_irq(unsigned long ret_addr,unsigned int irq, unsigned long old_irq_mask, unsigned long new_irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_enter_ack_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_exit_ack_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_enter_end_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_exit_end_irq(unsigned long ret_addr,unsigned int irq, unsigned long irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_enter_set_affinity_irq(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth) +{ +} +static void +prio_intr_default_diag_exit_set_affinity_irq(unsigned long ret_addr,unsigned int irq, unsigned long mask, unsigned long irq_mask, unsigned long depth) +{ +} +/* Preemption relevant events */ +static void +prio_intr_default_diag_preempt_disable(unsigned long ret_addr,unsigned long preempt_count) +{ +} +static void +prio_intr_default_diag_preempt_enable(unsigned long ret_addr,unsigned long preempt_count) +{ +} +static void +prio_intr_default_diag_preempt_enable_no_resched(unsigned long ret_addr,unsigned long preempt_count) +{ +} +static void +prio_intr_default_diag_preempt_check_resched(unsigned long ret_addr,unsigned long tif) +{ +} +static void +prio_intr_default_diag_syscall_enter(struct pt_regs *regs) +{ +} +static void +prio_intr_default_diag_syscall_exit(struct pt_regs *regs) +{ +} +static void +prio_intr_default_diag_irq_enter(struct pt_regs *regs) +{ +} +static void +prio_intr_default_diag_irq_exit(struct pt_regs *regs) +{ +} +static void +prio_intr_default_diag_restore(struct pt_regs *regs) +{ +} +static void +prio_intr_default_diag_page_fault(struct pt_regs *regs, unsigned long address, unsigned long ecode,unsigned long type) +{ +} +static void +prio_intr_default_diag_schedule(struct task_struct *prev,struct task_struct *next) +{ +} + +/* + * In kernel trace function + */ +void +prio_intr_diag_syscall_enter(struct pt_regs *regs) +{ + prio_intr_invoke_diag_func(diag_syscall_enter,regs); +} +void +prio_intr_diag_syscall_exit(struct pt_regs *regs) +{ + prio_intr_invoke_diag_func(diag_syscall_exit,regs); +} +void +prio_intr_diag_restore(struct pt_regs *regs) +{ + prio_intr_invoke_diag_func(diag_restore,regs); +} + +/* Wrapper functions for tracing. */ +void prio_intr_local_interrupt_save_wrapper(unsigned long oldmsr, unsigned long newmsr) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_local_interrupt_save(lr,oldmsr,newmsr); +} +void prio_intr_local_interrupt_restore_wrapper(unsigned long oldmsr, unsigned long newmsr) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_local_interrupt_restore(lr,oldmsr,newmsr); +} +void prio_intr_local_irq_save_wrapper(unsigned long oldmsr, unsigned long newmsr) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_local_irq_save(lr,oldmsr,newmsr); +} +void prio_intr_local_irq_restore_wrapper(unsigned long oldmsr, unsigned long newmsr) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_local_irq_restore(lr,oldmsr,newmsr); +} +void prio_intr_local_irq_enable_wrapper(void) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_local_irq_enable(lr); +} +void prio_intr_local_irq_disable_wrapper(void) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_local_irq_disable(lr); +} +void prio_intr_preempt_disable_wrapper(unsigned long preempt_count) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_preempt_disable(lr,preempt_count); +} +void prio_intr_preempt_enable_no_resched_wrapper(unsigned long preempt_count) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_preempt_enable_no_resched(lr,preempt_count); +} +void prio_intr_preempt_check_resched_wrapper(unsigned long tif) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_preempt_check_resched(lr,tif); +} +void prio_intr_preempt_enable_wrapper(unsigned long preempt_count) +{ + unsigned long lr; + + get_lr(lr); + prio_intr_diag_preempt_enable(lr,preempt_count); +} + +static prio_intr_diag_operations_t prio_intr_default_diag_ops={ + .diag_assert = prio_intr_default_diag_assert, + /* CPU level irq mask operations */ + .diag_local_irq_disable= prio_intr_default_diag_local_irq_disable, + .diag_local_irq_enable= prio_intr_default_diag_local_irq_enable, + .diag_local_irq_save= prio_intr_default_diag_local_irq_save, + .diag_local_irq_restore = prio_intr_default_diag_local_irq_restore, + .diag_local_interrupt_save = prio_intr_default_diag_local_interrupt_save, + .diag_local_interrupt_restore = prio_intr_default_diag_local_interrupt_restore, + /* interrupt mask operations */ + .diag_startup_irq = prio_intr_default_diag_startup_irq, + .diag_shutdown_irq = prio_intr_default_diag_shutdown_irq, + .diag_enable_irq = prio_intr_default_diag_enable_irq, + .diag_disable_irq = prio_intr_default_diag_disable_irq, + .diag_enter_ack_irq = prio_intr_default_diag_enter_ack_irq, + .diag_exit_ack_irq = prio_intr_default_diag_exit_ack_irq, + .diag_enter_end_irq = prio_intr_default_diag_enter_end_irq, + .diag_exit_end_irq = prio_intr_default_diag_exit_end_irq, + .diag_enter_set_affinity_irq = prio_intr_default_diag_enter_set_affinity_irq, + .diag_exit_set_affinity_irq = prio_intr_default_diag_exit_set_affinity_irq, + /* Preemption relevant events */ + .diag_preempt_disable = prio_intr_default_diag_preempt_disable, + .diag_preempt_enable = prio_intr_default_diag_preempt_enable, + .diag_preempt_enable_no_resched = prio_intr_default_diag_preempt_enable_no_resched, + .diag_preempt_check_resched = prio_intr_default_diag_preempt_check_resched, + /* System Calls */ + .diag_syscall_enter = prio_intr_default_diag_syscall_enter, + .diag_syscall_exit = prio_intr_default_diag_syscall_exit, + .diag_irq_enter = prio_intr_default_diag_irq_enter, + .diag_irq_exit = prio_intr_default_diag_irq_exit, + .diag_restore = prio_intr_default_diag_restore, + /* Exceptions */ + .diag_page_fault = prio_intr_default_diag_page_fault, + .diag_schedule = prio_intr_default_diag_schedule, +}; +prio_intr_diag_operations_t *prio_intr_diag_ops = &prio_intr_default_diag_ops; + +#define set_to_dummy_if_null(ops, function) \ + do { \ + if (!ops->function) { \ + ops->function = prio_intr_default_##function; \ + printk(KERN_ALERT "Override the " #function \ + " operation with the default one.\n");\ + } \ + } while (0) + + +static void fixup_diag_operations(prio_intr_diag_operations_t *ops) +{ + set_to_dummy_if_null(ops,diag_assert); + /* CPU level irq mask operations */ + set_to_dummy_if_null(ops,diag_local_irq_disable); + set_to_dummy_if_null(ops,diag_local_irq_enable); + set_to_dummy_if_null(ops,diag_local_irq_save); + set_to_dummy_if_null(ops,diag_local_irq_restore); + set_to_dummy_if_null(ops,diag_local_interrupt_save); + set_to_dummy_if_null(ops,diag_local_interrupt_restore); + /* interrupt mask operations */ + set_to_dummy_if_null(ops,diag_startup_irq); + set_to_dummy_if_null(ops,diag_shutdown_irq); + set_to_dummy_if_null(ops,diag_enable_irq); + set_to_dummy_if_null(ops,diag_disable_irq); + set_to_dummy_if_null(ops,diag_enter_ack_irq); + set_to_dummy_if_null(ops,diag_exit_ack_irq); + set_to_dummy_if_null(ops,diag_enter_end_irq); + set_to_dummy_if_null(ops,diag_exit_end_irq); + set_to_dummy_if_null(ops,diag_enter_set_affinity_irq); + set_to_dummy_if_null(ops,diag_exit_set_affinity_irq); + /* Preemption */ + set_to_dummy_if_null(ops,diag_preempt_disable); + set_to_dummy_if_null(ops,diag_preempt_enable); + set_to_dummy_if_null(ops,diag_preempt_enable_no_resched); + set_to_dummy_if_null(ops,diag_preempt_check_resched); + /* System call */ + set_to_dummy_if_null(ops,diag_syscall_enter); + set_to_dummy_if_null(ops,diag_syscall_exit); + set_to_dummy_if_null(ops,diag_irq_enter); + set_to_dummy_if_null(ops,diag_irq_exit); + set_to_dummy_if_null(ops,diag_restore); + /* Exception */ + set_to_dummy_if_null(ops,diag_page_fault); + set_to_dummy_if_null(ops,diag_schedule); + return ; +} +int +register_prio_intr_diag_operation(prio_intr_diag_operations_t *diag_ops) +{ + unsigned long flags; + + if (!diag_ops) + return -EINVAL; + + __in_diag_irq_save_flags(flags); + fixup_diag_operations(diag_ops); + prio_intr_diag_ops=diag_ops; + __in_diag_restore_flags(flags); + + return 0; +} + +int +unregister_prio_intr_diag_operation(prio_intr_diag_operations_t *diag_ops) +{ + unsigned long flags; + + if (prio_intr_diag_ops != diag_ops) + return -EINVAL; + + __in_diag_irq_save_flags(flags); + prio_intr_diag_ops=&prio_intr_default_diag_ops; + __in_diag_restore_flags(flags); + + return 0; +} +EXPORT_SYMBOL(register_prio_intr_diag_operation); +EXPORT_SYMBOL(unregister_prio_intr_diag_operation); diff -Naur linux-2.6.9/kernel/lvlintr.c linux-2.6.9-lvlintr/kernel/lvlintr.c --- linux-2.6.9/kernel/lvlintr.c 1970-01-01 09:00:00.000000000 +0900 +++ linux-2.6.9-lvlintr/kernel/lvlintr.c 2004-12-27 14:23:46.000000000 +0900 @@ -0,0 +1,396 @@ +/* + * arch/i386/kernel/lvlintr.c + * + * Author: Takeharu KATO (tkato@cs.fujitsu.co.jp) + * SUN Zhitai (sun@cs.fujitsu.co.jp) + * + * 2003 (c) Fujitsu Limited. This file is licensed under + * the terms of the GNU General Public License version 2. This program + * is licensed "as is" without any warranty of any kind, whether express + * or implied. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#undef PINTR_DBG_INFO +#undef PINTR_DBG_THREAD + +int irq2lvl[NR_IRQS]; +#if defined(CONFIG_PRIORITY_INTR_NORMAL) +static interrupt_thread_info_t intr_thread_info[NR_IRQS]; +#endif /* CONFIG_PRIORITY_INTR_NORMAL */ +unsigned long soft_mask_level=0; +static int __init setup_interrupt_level(void); + + +/**********************************************************************/ +/* function: convert_irq2level */ +/* Input: int irq ... IRQ number */ +/* Output: None */ +/* Return: Priority level */ +/* Note: None. */ +/* description: Convert irq to priority level. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Sep 25, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +convert_irq2level(int irq) +{ + return irq2lvl[irq2index(irq)]; +} +#if defined(CONFIG_PRIORITY_INTR_NORMAL) + +/**********************************************************************/ +/* function: lvlintrd */ +/* Input: void *data ... IRQ Number(It should be casted to int.) */ +/* Output: None */ +/* Return: 0 */ +/* Note: None. */ +/* description: Interrupt handling thread. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Sep 5, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static int +lvlintrd(void *data) +{ + struct sched_param __user param; + int irq=(int)(data); + int ret; + struct irqaction *action; + int status; + unsigned long flags,old_level; + irq_desc_t *desc = irq_desc + irq; + + local_irq_save(flags); + intr_thread_info[irq2index(irq)].irq_thread=current; + + param.sched_priority = irq2lvl[irq2index(irq)]; + + /* Make this thread daemon */ + daemonize("lvld/%d:%d", irq,irq2lvl[irq2index(irq)]); + /* This thread is kernel thread so access kernel datas */ + set_fs(KERNEL_DS); + + /* Set priority as interrupt priority */ + set_user_nice(current, 19); + ret = setscheduler(current->pid, SCHED_FIFO, ¶m); + /* I/O relevant thread */ + current->flags |= PF_NOFREEZE; +#ifdef PINTR_DBG_INFO + printk(KERN_DEBUG "Interrupt thread [Level: %d irq: %d tsk:0x%p]\n", + irq2lvl[irq2index(irq)],irq,intr_thread_info[irq2index(irq)].irq_thread); +#endif + /* + * Set up initial thread state + */ + set_current_state(TASK_INTERRUPTIBLE); + intr_thread_info[irq2index(irq)].state = PRIO_INTR_WAIT; + /* + * Handle interrupts + */ + for (;;) { + +#if defined(PINTR_DBG_THREAD) + printk(KERN_DEBUG "Sleep %d %x\n",irq,intr_thread_info[irq2index(irq)].irq_thread); +#endif + schedule(); /* Sleep on here */ + set_current_state(TASK_RUNNING); + intr_thread_info[irq2index(irq)].state=PRIO_INTR_ACTIVE; + spin_lock(&desc->lock); + status = desc->status; + + if (!(status & IRQ_DISABLED)) { + action = desc->action; + + for (;;) { + spin_unlock(&desc->lock); +#if defined(PINTR_DBG_THREAD) + printk(KERN_DEBUG "Call %d %x\n",irq,intr_thread_info[irq2index(irq)].irq_thread); +#endif /* PINTR_DBG_THREAD */ + old_level=soft_mask_level; + soft_mask_level=irq2lvl[irq2index(irq)]; + /* IRQ enabled in hanle_irq_event if it's needed. */ + HANDLE_IRQ_EVENT(irq, NULL, action); + soft_mask_level=old_level; + spin_lock(&desc->lock); + if (likely(!(desc->status & IRQ_PENDING))) + break; + desc->status &= ~IRQ_PENDING; + } + } + /* + * set thread state before unmask the irq just in case. + */ + local_irq_save(flags); + desc->status &= ~IRQ_INPROGRESS; + prio_intr_diag_assert((irqs_disabled())); + set_current_state(TASK_INTERRUPTIBLE); + intr_thread_info[irq2index(irq)].state = PRIO_INTR_WAIT; + set_tsk_need_resched(current); + + if (irq_desc[irq].handler) { + if (irq_desc[irq].handler->end) + irq_desc[irq].handler->end(irq); + else if (irq_desc[irq].handler->enable) + irq_desc[irq].handler->enable(irq); + } + spin_unlock(&desc->lock); + if (softirq_pending(smp_processor_id())) { + local_irq_restore(flags); + wakeup_softirqd(); + local_irq_save(flags); + set_tsk_need_resched(current); + } + } +} + +/**********************************************************************/ +/* function: wakeup_interrupt_thread */ +/* Input: int irq ... IRQ Number */ +/* Output: None */ +/* Return: -EBUSY ... The thread is running now. */ +/* 0 ... Sucess. */ +/* Note: None. */ +/* description: Wake up interrupt handling thread. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Sep 5, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +wakeup_interrupt_thread(int irq) +{ + unsigned long flags; + struct task_struct *tsk = intr_thread_info[irq2index(irq)].irq_thread; + + local_interrupt_save(flags); +#ifdef PINTR_DBG_THREAD + printk(KERN_DEBUG "irq %d wakeup tid 0x%x\n",irq,tsk); +#endif + if (likely(tsk && tsk->state != TASK_RUNNING)) { + wake_up_process(tsk); + set_tsk_need_resched(current); + } + local_interrupt_restore(flags); + return 0; +} +#endif /* PRIORITY_INTR_NORMAL */ + +/**********************************************************************/ +/* function: set_interrupt_mask */ +/* Input: level ... interrupt level to be set. */ +/* Output: old_level ... old interrupt level. */ +/* Return: 0 ... success */ +/* -EINVAL ... invalid level */ +/* Note: None. */ +/* description: set interrupt level. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Sep 5, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +set_interrupt_mask(unsigned long level,unsigned long *old_level) +{ + unsigned long flags; + unsigned long current_level,hwlvl; + struct sched_param __user param; + mm_segment_t old_segment; + int ret=0; + + if (level > MAX_INTR_LEVEL) + return -EINVAL; + + param.sched_priority = level; + + local_interrupt_save(flags); + + old_segment=get_fs(); + + current_level=soft_mask_level; + + hwlvl=get_hardmask_level(); + if (hwlvl) + current_level=hwpri2lvl(hwlvl); + + preempt_disable(); + + soft_mask_level=level; + + local_interrupt_restore(flags); + + if (!set_hardware_mask(current_level,level)) { + + set_fs(KERNEL_DS); // Set kernel data segment + + if (level) + ret=setscheduler(current->pid, SCHED_FIFO, ¶m); + else + ret=setscheduler(current->pid, SCHED_NORMAL, ¶m); + + set_fs(old_segment); // Restore user data segment + } + preempt_enable(); + + *old_level=current_level; + + return ret; +} + +#if defined(CONFIG_PRIORITY_INTR_NORMAL) +/**********************************************************************/ +/* function: interrupt_thread_init */ +/* Input: None */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Make interrupt handling thread */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Sep 5, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int __init +interrupt_thread_init(int cpu) +{ + int i=0; + int ret=0; + + for(i=0;iNR_IRQS) ) + continue; +#if !defined(CONFIG_PRIORITY_PRIVILEGED_IRQ) + if (get_intr_type(lvl) == PRIVILEGED_INTR) + panic("\n Invalid irq assignment is detected with irq=%3d level=%4d\n", +irq,lvl); +#endif /* CONFIG_PRIORITY_PRIVILEGED_IRQ */ + + irq2lvl[irq2index(irq)]=lvl; + } + kfree(mp); + +set_hard_lvl: + + arch_setup_lvlintr(); +#if defined(CONFIG_PRIORITY_INTR_NORMAL) + { + int rc; + + rc=interrupt_thread_init(smp_processor_id()); + if (rc<0) + panic("Interrupt thread init ERR!! return code=%d", rc); + } +#endif /* PRIORITY_INTR_NORMAL */ + +#if defined(PINTR_DBG_INFO) + printk(KERN_INFO PRIO_INTR_BANNER "\n"); + printk(KERN_INFO "IRQ Assignments:"); + for(i=0;i +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static privileged_irq_events_t privileged_irq_event_head={{&(privileged_irq_event_head.list),&(privileged_irq_event_head.list)},0,}; +static void __wakeup_proc_from_privileged_irq(void *arg); +static void __send_signal_from_privileged_irq(void *arg); + +/**********************************************************************/ +/* function: do_privileged_irq_request */ +/* Input: None */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Process queued requests. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void +do_privileged_irq_requests(void) +{ + privileged_irq_events_t *event; + struct list_head *tmp; + long flags; + struct thread_info *ti; + + ti = current_thread_info(); + + /* + * If there is a non-zero preempt_count or interrupts are disabled, + * we do not want to preempt the current task. Just return.. + */ + if (unlikely(ti->preempt_count)) { + return; + } + local_interrupt_save(flags); + while(!list_empty(&(privileged_irq_event_head.list))) { + tmp=(&(privileged_irq_event_head.list))->next; + list_del(tmp); + event=list_entry(tmp, privileged_irq_events_t , list); + + if (event->func) { + for(;event->count>0;--(event->count)) + event->func(event->arg); + } + } + local_interrupt_restore(flags); +} + +/**********************************************************************/ +/* function: enqueue_privileged_irq_request */ +/* Input: privileged_irq_events_t *event ... Pointer to the request. */ +/* Output: None */ +/* Return: request count */ +/* Note: None. */ +/* description: Enqueue privileged irq request */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +unsigned long +enqueue_privileged_irq_request(privileged_irq_events_t *event) +{ + long flags; + struct list_head *tmp; + + ++(event->count); + local_interrupt_save(flags); + list_for_each(tmp, &(privileged_irq_event_head.list)){ + privileged_irq_events_t *evntp=list_entry(tmp, privileged_irq_events_t , list); + if (evntp == event) { + local_interrupt_restore(flags); + return event->count; + } + } + list_add_tail(&(event->list), &(privileged_irq_event_head.list)); + local_interrupt_restore(flags); + + return event->count; +} + +/**********************************************************************/ +/* function: dequeue_privileged_irq_request */ +/* Input: privileged_irq_events_t *event ... Pointer to the request*/ +/* Output: None */ +/* Return: 0 Successfully deleted */ +/* -EBUSY(-16) */ +/* Note: None. */ +/* description: Enqueue privileged irq request */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +int +dequeue_privileged_irq_request(privileged_irq_events_t *event) +{ + long flags; + + local_interrupt_save(flags); + --event->count; + if (event->count) { + local_interrupt_restore(flags); + return -EBUSY; + } + list_del(&(event->list)); + local_interrupt_restore(flags); + return 0; +} + +/**********************************************************************/ +/* function: __wakeup_proc_from_privileged_irq */ +/* Input: void *arg ... Pointer to the process queue */ +/* which procs are stored. */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: wake up process. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static void __wakeup_proc_from_privileged_irq(void *arg) +{ + wait_queue_head_t *q=(wait_queue_head_t *)arg; + + wake_up_interruptible_sync(q); + set_tsk_need_resched(current); +} + +/**********************************************************************/ +/* function: __send_signal_from_privileged_irq */ +/* Input: void *arg ... Pointer to the signal information.*/ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Send the signal to a process. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +static void __send_signal_from_privileged_irq(void *arg) +{ + exp_sig_t *sig=(exp_sig_t *)arg; + + kill_proc(sig->pid, sig->signo, 0); + set_tsk_need_resched(current); +} + +/**********************************************************************/ +/* function: wake_up_interruptible_sync_from_privileged_irq */ +/* Input: wait_queue_head_t *q . Wait queue which procs are stored */ +/* privileged_irq_events_t *event ... event information */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: wake up process. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void wake_up_interruptible_sync_from_privileged_irq(wait_queue_head_t *q,privileged_irq_events_t *event){ + event->func=__wakeup_proc_from_privileged_irq; + event->arg=q; + event->count=0; + enqueue_privileged_irq_request(event); + + return; +} + +/**********************************************************************/ +/* function: send_signal_from_privileged_irq */ +/* Input: exp_sig_t *sig ... signal information */ +/* privileged_irq_events_t *event ... event information */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: send signal from critical interrupt. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ +void send_signal_from_privileged_irq(exp_sig_t *sig, privileged_irq_events_t *event){ + event->func=__send_signal_from_privileged_irq; + event->arg=sig; + event->count=0; + enqueue_privileged_irq_request(event); + + return; +} + +/**********************************************************************/ +/* function: setup_privileged_interrupt */ +/* Input: char *str ... No use */ +/* Output: None */ +/* Return: None */ +/* Note: None. */ +/* description: Initilize privileged interrupt facilities. */ +/* author: Takeharu KATO */ +/* Revision: V1.0(original), Aug 15, 2003 */ +/* Vx.xx (date and author) */ +/**********************************************************************/ + +static int __init +setup_privileged_interrupt(void) +{ + + return 0; +} +arch_initcall(setup_privileged_interrupt); +EXPORT_SYMBOL(wake_up_interruptible_sync_from_privileged_irq); +EXPORT_SYMBOL(send_signal_from_privileged_irq); +EXPORT_SYMBOL(do_privileged_irq_requests); +EXPORT_SYMBOL(enqueue_privileged_irq_request); +EXPORT_SYMBOL(dequeue_privileged_irq_request); diff -Naur linux-2.6.9/kernel/sched.c linux-2.6.9-lvlintr/kernel/sched.c --- linux-2.6.9/kernel/sched.c 2004-12-27 11:08:07.000000000 +0900 +++ linux-2.6.9-lvlintr/kernel/sched.c 2004-12-27 11:42:03.000000000 +0900 @@ -3180,7 +3180,11 @@ /* * setscheduler - change the scheduling policy and/or RT priority of a thread. */ +#if defined(CONFIG_PRIORITY_INTR) +int setscheduler(pid_t pid, int policy, struct sched_param __user *param) +#else static int setscheduler(pid_t pid, int policy, struct sched_param __user *param) +#endif /* CONFIG_PRIORITY_INTR */ { struct sched_param lp; int retval = -EINVAL; diff -Naur linux-2.6.9/kernel/softirq.c linux-2.6.9-lvlintr/kernel/softirq.c --- linux-2.6.9/kernel/softirq.c 2004-12-27 11:08:07.000000000 +0900 +++ linux-2.6.9-lvlintr/kernel/softirq.c 2004-12-27 14:23:46.000000000 +0900 @@ -16,6 +16,9 @@ #include #include #include +#if defined(CONFIG_PRIORITY_INTR) +#include +#endif /* CONFIG_PRIORITY_INTR */ #include /* @@ -44,14 +47,21 @@ static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp; static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); - +#if defined(CONFIG_PRIORITY_INTR_NORMAL) +static asmlinkage void __raw_do_softirq(void); +#endif /* CONFIG_PRIORITY_INTR_NORMAL */ /* * we cannot loop indefinitely here to avoid userspace starvation, * but we also don't want to introduce a worst case 1/HZ latency * to the pending events, so lets the scheduler to balance * the softirq load for us. */ +#if defined(CONFIG_PRIORITY_INTR) + /* config.h is included by module.h at least. */ +void wakeup_softirqd(void) +#else static inline void wakeup_softirqd(void) +#endif /* CONFIG_PRIORITY_INTR */ { /* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk = __get_cpu_var(ksoftirqd); @@ -104,16 +114,27 @@ pending = local_softirq_pending(); if (pending && --max_restart) goto restart; - +#if !defined(CONFIG_PRIORITY_INTR_NORMAL) if (pending) wakeup_softirqd(); - +#endif /* !CONFIG_PRIORITY_INTR */ __local_bh_enable(); } #ifndef __ARCH_HAS_DO_SOFTIRQ +#if defined(CONFIG_PRIORITY_INTR_NORMAL) +asmlinkage void do_softirq(void) +{ + wakeup_softirqd(); + set_tsk_need_resched(current); +} +#endif /* CONFIG_PRIORITY_INTR_NORMAL */ +#if defined(CONFIG_PRIORITY_INTR_NORMAL) +static asmlinkage void __raw_do_softirq(void) +#else asmlinkage void do_softirq(void) +#endif /* CONFIG_PRIORITY_INTR_NORMAL */ { __u32 pending; unsigned long flags; @@ -327,10 +348,56 @@ static int ksoftirqd(void * __bind_cpu) { +#if defined(CONFIG_PRIORITY_INTR_NORMAL) + struct sched_param __user param_high = { .sched_priority = CONFIG_PRIORITY_INTR_SOFTIRQ_LVL }; + struct sched_param __user param_low = { .sched_priority = 0 }; +#endif /* CONFIG_PRIORITY_INTR_NORMAL */ + set_user_nice(current, 19); current->flags |= PF_NOFREEZE; set_current_state(TASK_INTERRUPTIBLE); +#if defined(CONFIG_PRIORITY_INTR_NORMAL) + + if (CONFIG_PRIORITY_INTR_SOFTIRQ_LVL) + setscheduler(current->pid, SCHED_FIFO, ¶m_high); + + for (;;) { + if (!local_softirq_pending()) { + end_soft_irq: + prio_intr_diag_assert( (current->policy==SCHED_FIFO) && + (current->rt_priority==CONFIG_PRIORITY_INTR_SOFTIRQ_LVL)); + __set_current_state(TASK_INTERRUPTIBLE); + schedule(); + } + __set_current_state(TASK_RUNNING); + + prio_intr_diag_assert( (current->policy==SCHED_FIFO) && + (current->rt_priority==CONFIG_PRIORITY_INTR_SOFTIRQ_LVL)); + __raw_do_softirq(); + cond_resched(); + + if (!local_softirq_pending()) + goto end_soft_irq; + + /* Enter NORMAL_SCHED */ + setscheduler(current->pid, SCHED_NORMAL, ¶m_low); + prio_intr_diag_assert(current->policy==SCHED_NORMAL); + + do{ + __raw_do_softirq(); + if (need_resched()) { + if (CONFIG_PRIORITY_INTR_SOFTIRQ_LVL) + setscheduler(current->pid, SCHED_FIFO, ¶m_high); + goto end_soft_irq; + } + }while (local_softirq_pending()); + + /* Return to FIFO_SCHED */ + if (CONFIG_PRIORITY_INTR_SOFTIRQ_LVL) + setscheduler(current->pid, SCHED_FIFO, ¶m_high); + } +#else while (!kthread_should_stop()) { if (!local_softirq_pending()) @@ -365,6 +432,7 @@ } __set_current_state(TASK_RUNNING); return 0; +#endif /* CONFIG_PRIORITY_INTR_NORMAL */ } #ifdef CONFIG_HOTPLUG_CPU diff -Naur linux-2.6.9/lib/Kconfig.debug linux-2.6.9-lvlintr/lib/Kconfig.debug --- linux-2.6.9/lib/Kconfig.debug 2004-12-27 11:08:08.000000000 +0900 +++ linux-2.6.9-lvlintr/lib/Kconfig.debug 2004-12-27 11:42:03.000000000 +0900 @@ -47,7 +47,7 @@ config DEBUG_SPINLOCK_SLEEP bool "Sleep-inside-spinlock checking" - depends on DEBUG_KERNEL && (X86 || IA64 || MIPS || PPC32 || PPC64 || ARCH_S390 || SPARC32 || SPARC64) + depends on DEBUG_KERNEL && (X86 || IA64 || MIPS || PPC32 || PPC64 || ARCH_S390 || SPARC32 || SPARC64) && !PRIORITY_INTR help If you say Y here, various routines which may sleep will become very noisy if they are called with a spinlock held. diff -Naur linux-2.6.9/net/ipv4/ipconfig.c linux-2.6.9-lvlintr/net/ipv4/ipconfig.c --- linux-2.6.9/net/ipv4/ipconfig.c 2004-12-27 11:07:29.000000000 +0900 +++ linux-2.6.9-lvlintr/net/ipv4/ipconfig.c 2004-12-28 16:12:38.000000000 +0900 @@ -1102,6 +1102,9 @@ jiff = jiffies + (d->next ? CONF_INTER_TIMEOUT : timeout); while (time_before(jiffies, jiff) && !ic_got_reply) { +#if defined(CONFIG_PRIORITY_INTR) + schedule(); +#endif /* CONFIG_PRIORITY_INTR */ barrier(); cpu_relax(); } @@ -1262,7 +1265,14 @@ /* Give hardware a chance to settle */ jiff = jiffies + CONF_PRE_OPEN; while (time_before(jiffies, jiff)) - cpu_relax(); +#if defined(CONFIG_PRIORITY_INTR) + { + schedule(); + cpu_relax(); + } +#else + cpu_relax(); +#endif /* CONFIG_PRIORITY_INTR */ /* Setup all network devices */ if (ic_open_devs() < 0) @@ -1271,7 +1281,14 @@ /* Give drivers a chance to settle */ jiff = jiffies + CONF_POST_OPEN; while (time_before(jiffies, jiff)) - cpu_relax(); +#if defined(CONFIG_PRIORITY_INTR) + { + schedule(); + cpu_relax(); + } +#else + cpu_relax(); +#endif /* CONFIG_PRIORITY_INTR */ /* * If the config information is insufficient (e.g., our IP address or