diff options
author | David Daney <ddaney@caviumnetworks.com> | 2011-03-25 12:38:51 -0700 |
---|---|---|
committer | Thomas Gleixner <tglx@linutronix.de> | 2011-03-29 14:48:06 +0200 |
commit | 0c3263870f5f80cf7b6cb322bd8e708ce568d36b (patch) | |
tree | b1e2ed41a6b9b10a16fdb290f6f54cb148ccaf1a /arch | |
parent | a458465641bf61a00f4ca54da7265202a911f975 (diff) |
MIPS: Octeon: Rewrite interrupt handling code.
This includes conversion to new style irq_chip functions, and
correctly enabling/disabling per-CPU interrupts.
The hardware interrupt bit to irq number mapping is now done with a
flexible map, instead of by bit twiddling the irq number.
[ tglx: Adjusted to new irq_cpu_on/offline callbacks and
__irq_set_affinity_lock ]
Signed-off-by: David Daney <ddaney@caviumnetworks.com>
Cc: linux-mips@linux-mips.org
Cc: ralf@linux-mips.org
LKML-Reference: <1301081931-11240-5-git-send-email-ddaney@caviumnetworks.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Diffstat (limited to 'arch')
-rw-r--r-- | arch/mips/cavium-octeon/octeon-irq.c | 1422 | ||||
-rw-r--r-- | arch/mips/cavium-octeon/setup.c | 12 | ||||
-rw-r--r-- | arch/mips/cavium-octeon/smp.c | 39 | ||||
-rw-r--r-- | arch/mips/include/asm/mach-cavium-octeon/irq.h | 243 | ||||
-rw-r--r-- | arch/mips/include/asm/octeon/octeon.h | 2 | ||||
-rw-r--r-- | arch/mips/pci/msi-octeon.c | 20 |
6 files changed, 921 insertions, 817 deletions
diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index ce7500cdf5b7..03c081da3df4 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -3,10 +3,13 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. * - * Copyright (C) 2004-2008, 2009, 2010 Cavium Networks + * Copyright (C) 2004-2008, 2009, 2010, 2011 Cavium Networks */ -#include <linux/irq.h> + #include <linux/interrupt.h> +#include <linux/bitops.h> +#include <linux/percpu.h> +#include <linux/irq.h> #include <linux/smp.h> #include <asm/octeon/octeon.h> @@ -14,6 +17,47 @@ static DEFINE_RAW_SPINLOCK(octeon_irq_ciu0_lock); static DEFINE_RAW_SPINLOCK(octeon_irq_ciu1_lock); +static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror); +static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror); + +static __read_mostly u8 octeon_irq_ciu_to_irq[8][64]; + +union octeon_ciu_chip_data { + void *p; + unsigned long l; + struct { + unsigned int line:6; + unsigned int bit:6; + } s; +}; + +struct octeon_core_chip_data { + struct mutex core_irq_mutex; + bool current_en; + bool desired_en; + u8 bit; +}; + +#define MIPS_CORE_IRQ_LINES 8 + +static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES]; + +static void __init octeon_irq_set_ciu_mapping(int irq, int line, int bit, + struct irq_chip *chip, + irq_flow_handler_t handler) +{ + union octeon_ciu_chip_data cd; + + irq_set_chip_and_handler(irq, chip, handler); + + cd.l = 0; + cd.s.line = line; + cd.s.bit = bit; + + irq_set_chip_data(irq, cd.p); + octeon_irq_ciu_to_irq[line][bit] = irq; +} + static int octeon_coreid_for_cpu(int cpu) { #ifdef CONFIG_SMP @@ -23,9 +67,20 @@ static int octeon_coreid_for_cpu(int cpu) #endif } -static void octeon_irq_core_ack(unsigned int irq) +static int octeon_cpu_for_coreid(int coreid) +{ +#ifdef CONFIG_SMP + return cpu_number_map(coreid); +#else + return smp_processor_id(); +#endif +} + +static void octeon_irq_core_ack(struct irq_data *data) { - unsigned int bit = irq - OCTEON_IRQ_SW0; + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + unsigned int bit = cd->bit; + /* * We don't need to disable IRQs to make these atomic since * they are already disabled earlier in the low level @@ -37,131 +92,133 @@ static void octeon_irq_core_ack(unsigned int irq) clear_c0_cause(0x100 << bit); } -static void octeon_irq_core_eoi(unsigned int irq) +static void octeon_irq_core_eoi(struct irq_data *data) { - struct irq_desc *desc = irq_to_desc(irq); - unsigned int bit = irq - OCTEON_IRQ_SW0; - /* - * If an IRQ is being processed while we are disabling it the - * handler will attempt to unmask the interrupt after it has - * been disabled. - */ - if ((unlikely(desc->status & IRQ_DISABLED))) - return; + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + /* * We don't need to disable IRQs to make these atomic since * they are already disabled earlier in the low level * interrupt code. */ - set_c0_status(0x100 << bit); + set_c0_status(0x100 << cd->bit); } -static void octeon_irq_core_enable(unsigned int irq) +static void octeon_irq_core_set_enable_local(void *arg) { - unsigned long flags; - unsigned int bit = irq - OCTEON_IRQ_SW0; + struct irq_data *data = arg; + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + unsigned int mask = 0x100 << cd->bit; /* - * We need to disable interrupts to make sure our updates are - * atomic. + * Interrupts are already disabled, so these are atomic. */ - local_irq_save(flags); - set_c0_status(0x100 << bit); - local_irq_restore(flags); + if (cd->desired_en) + set_c0_status(mask); + else + clear_c0_status(mask); + } -static void octeon_irq_core_disable_local(unsigned int irq) +static void octeon_irq_core_disable(struct irq_data *data) { - unsigned long flags; - unsigned int bit = irq - OCTEON_IRQ_SW0; - /* - * We need to disable interrupts to make sure our updates are - * atomic. - */ - local_irq_save(flags); - clear_c0_status(0x100 << bit); - local_irq_restore(flags); + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + cd->desired_en = false; } -static void octeon_irq_core_disable(unsigned int irq) +static void octeon_irq_core_enable(struct irq_data *data) { -#ifdef CONFIG_SMP - on_each_cpu((void (*)(void *)) octeon_irq_core_disable_local, - (void *) (long) irq, 1); -#else - octeon_irq_core_disable_local(irq); -#endif + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + cd->desired_en = true; } -static struct irq_chip octeon_irq_chip_core = { - .name = "Core", - .enable = octeon_irq_core_enable, - .disable = octeon_irq_core_disable, - .ack = octeon_irq_core_ack, - .eoi = octeon_irq_core_eoi, -}; +static void octeon_irq_core_bus_lock(struct irq_data *data) +{ + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + mutex_lock(&cd->core_irq_mutex); +} -static void octeon_irq_ciu0_ack(unsigned int irq) +static void octeon_irq_core_bus_sync_unlock(struct irq_data *data) { - switch (irq) { - case OCTEON_IRQ_GMX_DRP0: - case OCTEON_IRQ_GMX_DRP1: - case OCTEON_IRQ_IPD_DRP: - case OCTEON_IRQ_KEY_ZERO: - case OCTEON_IRQ_TIMER0: - case OCTEON_IRQ_TIMER1: - case OCTEON_IRQ_TIMER2: - case OCTEON_IRQ_TIMER3: - { - int index = cvmx_get_core_num() * 2; - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); - /* - * CIU timer type interrupts must be acknoleged by - * writing a '1' bit to their sum0 bit. - */ - cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); - break; - } - default: - break; + struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data); + + if (cd->desired_en != cd->current_en) { + on_each_cpu(octeon_irq_core_set_enable_local, data, 1); + + cd->current_en = cd->desired_en; } - /* - * In order to avoid any locking accessing the CIU, we - * acknowledge CIU interrupts by disabling all of them. This - * way we can use a per core register and avoid any out of - * core locking requirements. This has the side affect that - * CIU interrupts can't be processed recursively. - * - * We don't need to disable IRQs to make these atomic since - * they are already disabled earlier in the low level - * interrupt code. - */ - clear_c0_status(0x100 << 2); + mutex_unlock(&cd->core_irq_mutex); } -static void octeon_irq_ciu0_eoi(unsigned int irq) + +static void octeon_irq_core_cpu_online(struct irq_data *data) { - /* - * Enable all CIU interrupts again. We don't need to disable - * IRQs to make these atomic since they are already disabled - * earlier in the low level interrupt code. - */ - set_c0_status(0x100 << 2); + if (irqd_irq_disabled(data)) + octeon_irq_core_eoi(data); +} + +static void octeon_irq_core_cpu_offline(struct irq_data *data) +{ + if (irqd_irq_disabled(data)) + octeon_irq_core_ack(data); } -static int next_coreid_for_irq(struct irq_desc *desc) +static struct irq_chip octeon_irq_chip_core = { + .name = "Core", + .irq_enable = octeon_irq_core_enable, + .irq_disable = octeon_irq_core_disable, + .irq_ack = octeon_irq_core_ack, + .irq_eoi = octeon_irq_core_eoi, + .irq_bus_lock = octeon_irq_core_bus_lock, + .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock, + + .irq_cpu_online = octeon_irq_core_cpu_online, + .irq_cpu_offline = octeon_irq_core_cpu_offline, +}; + +static void __init octeon_irq_init_core(void) +{ + int i; + int irq; + struct octeon_core_chip_data *cd; + + for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) { + cd = &octeon_irq_core_chip_data[i]; + cd->current_en = false; + cd->desired_en = false; + cd->bit = i; + mutex_init(&cd->core_irq_mutex); + + irq = OCTEON_IRQ_SW0 + i; + switch (irq) { + case OCTEON_IRQ_TIMER: + case OCTEON_IRQ_SW0: + case OCTEON_IRQ_SW1: + case OCTEON_IRQ_5: + case OCTEON_IRQ_PERF: + irq_set_chip_data(irq, cd); + irq_set_chip_and_handler(irq, &octeon_irq_chip_core, + handle_percpu_irq); + break; + default: + break; + } + } +} + +static int next_cpu_for_irq(struct irq_data *data) { #ifdef CONFIG_SMP - int coreid; - int weight = cpumask_weight(desc->affinity); + int cpu; + int weight = cpumask_weight(data->affinity); if (weight > 1) { - int cpu = smp_processor_id(); + cpu = smp_processor_id(); for (;;) { - cpu = cpumask_next(cpu, desc->affinity); + cpu = cpumask_next(cpu, data->affinity); if (cpu >= nr_cpu_ids) { cpu = -1; continue; @@ -169,83 +226,175 @@ static int next_coreid_for_irq(struct irq_desc *desc) break; } } - coreid = octeon_coreid_for_cpu(cpu); } else if (weight == 1) { - coreid = octeon_coreid_for_cpu(cpumask_first(desc->affinity)); + cpu = cpumask_first(data->affinity); } else { - coreid = cvmx_get_core_num(); + cpu = smp_processor_id(); } - return coreid; + return cpu; #else - return cvmx_get_core_num(); + return smp_processor_id(); #endif } -static void octeon_irq_ciu0_enable(unsigned int irq) +static void octeon_irq_ciu_enable(struct irq_data *data) { - struct irq_desc *desc = irq_to_desc(irq); - int coreid = next_coreid_for_irq(desc); + int cpu = next_cpu_for_irq(data); + int coreid = octeon_coreid_for_cpu(cpu); + unsigned long *pen; unsigned long flags; - uint64_t en0; - int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */ + union octeon_ciu_chip_data cd; + + cd.p = irq_data_get_irq_chip_data(data); - raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); - en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)); - en0 |= 1ull << bit; - cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0); - cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)); - raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + if (cd.s.line == 0) { + raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); + pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); + raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + } else { + raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); + pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); + raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); + } } -static void octeon_irq_ciu0_enable_mbox(unsigned int irq) +static void octeon_irq_ciu_enable_local(struct irq_data *data) +{ + unsigned long *pen; + unsigned long flags; + union octeon_ciu_chip_data cd; + + cd.p = irq_data_get_irq_chip_data(data); + + if (cd.s.line == 0) { + raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); + pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror); + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); + raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + } else { + raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); + pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror); + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); + raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); + } +} + +static void octeon_irq_ciu_disable_local(struct irq_data *data) +{ + unsigned long *pen; + unsigned long flags; + union octeon_ciu_chip_data cd; + + cd.p = irq_data_get_irq_chip_data(data); + + if (cd.s.line == 0) { + raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); + pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror); + clear_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen); + raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + } else { + raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); + pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror); + clear_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen); + raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); + } +} + +static void octeon_irq_ciu_disable_all(struct irq_data *data) { - int coreid = cvmx_get_core_num(); unsigned long flags; - uint64_t en0; - int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */ + unsigned long *pen; + int cpu; + union octeon_ciu_chip_data cd; + + wmb(); /* Make sure flag changes arrive before register updates. */ - raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); - en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)); - en0 |= 1ull << bit; - cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0); - cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)); - raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + cd.p = irq_data_get_irq_chip_data(data); + + if (cd.s.line == 0) { + raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); + for_each_online_cpu(cpu) { + int coreid = octeon_coreid_for_cpu(cpu); + pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); + clear_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); + } + raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + } else { + raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); + for_each_online_cpu(cpu) { + int coreid = octeon_coreid_for_cpu(cpu); + pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); + clear_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); + } + raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); + } } -static void octeon_irq_ciu0_disable(unsigned int irq) +static void octeon_irq_ciu_enable_all(struct irq_data *data) { - int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */ unsigned long flags; - uint64_t en0; + unsigned long *pen; int cpu; - raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); - for_each_online_cpu(cpu) { - int coreid = octeon_coreid_for_cpu(cpu); - en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)); - en0 &= ~(1ull << bit); - cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0); + union octeon_ciu_chip_data cd; + + cd.p = irq_data_get_irq_chip_data(data); + + if (cd.s.line == 0) { + raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); + for_each_online_cpu(cpu) { + int coreid = octeon_coreid_for_cpu(cpu); + pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); + } + raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + } else { + raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); + for_each_online_cpu(cpu) { + int coreid = octeon_coreid_for_cpu(cpu); + pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); + } + raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); } - /* - * We need to do a read after the last update to make sure all - * of them are done. - */ - cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2)); - raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); } /* * Enable the irq on the next core in the affinity set for chips that * have the EN*_W1{S,C} registers. */ -static void octeon_irq_ciu0_enable_v2(unsigned int irq) +static void octeon_irq_ciu_enable_v2(struct irq_data *data) { - int index; - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); - struct irq_desc *desc = irq_to_desc(irq); + u64 mask; + int cpu = next_cpu_for_irq(data); + union octeon_ciu_chip_data cd; - if ((desc->status & IRQ_DISABLED) == 0) { - index = next_coreid_for_irq(desc) * 2; + cd.p = irq_data_get_irq_chip_data(data); + mask = 1ull << (cd.s.bit); + + /* + * Called under the desc lock, so these should never get out + * of sync. + */ + if (cd.s.line == 0) { + int index = octeon_coreid_for_cpu(cpu) * 2; + set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); + } else { + int index = octeon_coreid_for_cpu(cpu) * 2 + 1; + set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); } } @@ -253,83 +402,180 @@ static void octeon_irq_ciu0_enable_v2(unsigned int irq) * Enable the irq on the current CPU for chips that * have the EN*_W1{S,C} registers. */ -static void octeon_irq_ciu0_enable_mbox_v2(unsigned int irq) +static void octeon_irq_ciu_enable_local_v2(struct irq_data *data) { - int index; - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); + u64 mask; + union octeon_ciu_chip_data cd; + + cd.p = irq_data_get_irq_chip_data(data); + mask = 1ull << (cd.s.bit); - index = cvmx_get_core_num() * 2; - cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); + if (cd.s.line == 0) { + int index = cvmx_get_core_num() * 2; + set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror)); + cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); + } else { + int index = cvmx_get_core_num() * 2 + 1; + set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror)); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); + } +} + +static void octeon_irq_ciu_disable_local_v2(struct irq_data *data) +{ + u64 mask; + union octeon_ciu_chip_data cd; + + cd.p = irq_data_get_irq_chip_data(data); + mask = 1ull << (cd.s.bit); + + if (cd.s.line == 0) { + int index = cvmx_get_core_num() * 2; + clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror)); + cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); + } else { + int index = cvmx_get_core_num() * 2 + 1; + clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror)); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); + } } /* - * Disable the irq on the current core for chips that have the EN*_W1{S,C} - * registers. + * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq. */ -static void octeon_irq_ciu0_ack_v2(unsigned int irq) -{ - int index = cvmx_get_core_num() * 2; - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); - - switch (irq) { - case OCTEON_IRQ_GMX_DRP0: - case OCTEON_IRQ_GMX_DRP1: - case OCTEON_IRQ_IPD_DRP: - case OCTEON_IRQ_KEY_ZERO: - case OCTEON_IRQ_TIMER0: - case OCTEON_IRQ_TIMER1: - case OCTEON_IRQ_TIMER2: - case OCTEON_IRQ_TIMER3: - /* - * CIU timer type interrupts must be acknoleged by - * writing a '1' bit to their sum0 bit. - */ +static void octeon_irq_ciu_ack(struct irq_data *data) +{ + u64 mask; + union octeon_ciu_chip_data cd; + + cd.p = data->chip_data; + mask = 1ull << (cd.s.bit); + + if (cd.s.line == 0) { + int index = cvmx_get_core_num() * 2; cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask); - break; - default: - break; + } else { + cvmx_write_csr(CVMX_CIU_INT_SUM1, mask); } - - cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); } /* - * Enable the irq on the current core for chips that have the EN*_W1{S,C} + * Disable the irq on the all cores for chips that have the EN*_W1{S,C} * registers. */ -static void octeon_irq_ciu0_eoi_mbox_v2(unsigned int irq) +static void octeon_irq_ciu_disable_all_v2(struct irq_data *data) { - struct irq_desc *desc = irq_to_desc(irq); - int index = cvmx_get_core_num() * 2; - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); + int cpu; + u64 mask; + union octeon_ciu_chip_data cd; - if (likely((desc->status & IRQ_DISABLED) == 0)) - cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); + wmb(); /* Make sure flag changes arrive before register updates. */ + + cd.p = data->chip_data; + mask = 1ull << (cd.s.bit); + + if (cd.s.line == 0) { + for_each_online_cpu(cpu) { + int index = octeon_coreid_for_cpu(cpu) * 2; + clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); + cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); + } + } else { + for_each_online_cpu(cpu) { + int index = octeon_coreid_for_cpu(cpu) * 2 + 1; + clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); + } + } } /* - * Disable the irq on the all cores for chips that have the EN*_W1{S,C} + * Enable the irq on the all cores for chips that have the EN*_W1{S,C} * registers. */ -static void octeon_irq_ciu0_disable_all_v2(unsigned int irq) +static void octeon_irq_ciu_enable_all_v2(struct irq_data *data) { - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); - int index; int cpu; - for_each_online_cpu(cpu) { - index = octeon_coreid_for_cpu(cpu) * 2; - cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); + u64 mask; + union octeon_ciu_chip_data cd; + + cd.p = data->chip_data; + mask = 1ull << (cd.s.bit); + + if (cd.s.line == 0) { + for_each_online_cpu(cpu) { + int index = octeon_coreid_for_cpu(cpu) * 2; + set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu)); + cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); + } + } else { + for_each_online_cpu(cpu) { + int index = octeon_coreid_for_cpu(cpu) * 2 + 1; + set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu)); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); + } } } +static void octeon_irq_cpu_online_mbox(struct irq_data *data) +{ + if (irqd_irq_disabled(data)) + octeon_irq_ciu_enable_local(data); +} + +static void octeon_irq_cpu_online_mbox_v2(struct irq_data *data) +{ + if (irqd_irq_disabled(data)) + octeon_irq_ciu_enable_local_v2(data); +} + +static void octeon_irq_cpu_offline_mbox(struct irq_data *data) +{ + if (irqd_irq_disabled(data)) + octeon_irq_ciu_disable_local(data); +} + +static void octeon_irq_cpu_offline_mbox_v2(struct irq_data *data) +{ + if (irqd_irq_disabled(data)) + octeon_irq_ciu_disable_local_v2(data); +} + #ifdef CONFIG_SMP -static int octeon_irq_ciu0_set_affinity(unsigned int irq, const struct cpumask *dest) + +static void octeon_irq_cpu_offline_ciu(struct irq_data *data) +{ + int cpu = smp_processor_id(); + cpumask_t new_affinity; + + if (!cpumask_test_cpu(cpu, data->affinity)) + return; + + if (cpumask_weight(data->affinity) > 1) { + /* + * It has multi CPU affinity, just remove this CPU + * from the affinity set. + */ + cpumask_copy(&new_affinity, data->affinity); + cpumask_clear_cpu(cpu, &new_affinity); + } else { + /* Otherwise, put it on lowest numbered online CPU. */ + cpumask_clear(&new_affinity); + cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); + } + __irq_set_affinity_locked(data, &new_affinity); +} + +static int octeon_irq_ciu_set_affinity(struct irq_data *data, + const struct cpumask *dest, bool force) { int cpu; - struct irq_desc *desc = irq_to_desc(irq); + struct irq_desc *desc = irq_to_desc(data->irq); int enable_one = (desc->status & IRQ_DISABLED) == 0; unsigned long flags; - int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */ + union octeon_ciu_chip_data cd; + + cd.p = data->chip_data; /* * For non-v2 CIU, we will allow only single CPU affinity. @@ -339,26 +585,40 @@ static int octeon_irq_ciu0_set_affinity(unsigned int irq, const struct cpumask * if (cpumask_weight(dest) != 1) return -EINVAL; - raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); - for_each_online_cpu(cpu) { - int coreid = octeon_coreid_for_cpu(cpu); - uint64_t en0 = - cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)); - if (cpumask_test_cpu(cpu, dest) && enable_one) { - enable_one = 0; - en0 |= 1ull << bit; - } else { - en0 &= ~(1ull << bit); + if (desc->status & IRQ_DISABLED) + return 0; + + if (cd.s.line == 0) { + raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags); + for_each_online_cpu(cpu) { + int coreid = octeon_coreid_for_cpu(cpu); + unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); + + if (cpumask_test_cpu(cpu, dest) && enable_one) { + enable_one = 0; + set_bit(cd.s.bit, pen); + } else { + clear_bit(cd.s.bit, pen); + } + cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen); + } + raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); + } else { + raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); + for_each_online_cpu(cpu) { + int coreid = octeon_coreid_for_cpu(cpu); + unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); + + if (cpumask_test_cpu(cpu, dest) && enable_one) { + enable_one = 0; + set_bit(cd.s.bit, pen); + } else { + clear_bit(cd.s.bit, pen); + } + cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); } - cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0); + raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); } - /* - * We need to do a read after the last update to make sure all - * of them are done. - */ - cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2)); - raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags); - return 0; } @@ -366,22 +626,47 @@ static int octeon_irq_ciu0_set_affinity(unsigned int irq, const struct cpumask * * Set affinity for the irq for chips that have the EN*_W1{S,C} * registers. */ -static int octeon_irq_ciu0_set_affinity_v2(unsigned int irq, - const struct cpumask *dest) +static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data, + const struct cpumask *dest, + bool force) { int cpu; - int index; - struct irq_desc *desc = irq_to_desc(irq); + struct irq_desc *desc = irq_to_desc(data->irq); int enable_one = (desc->status & IRQ_DISABLED) == 0; - u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0); - - for_each_online_cpu(cpu) { - index = octeon_coreid_for_cpu(cpu) * 2; - if (cpumask_test_cpu(cpu, dest) && enable_one) { - enable_one = 0; - cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); - } else { - cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); + u64 mask; + union octeon_ciu_chip_data cd; + + if (desc->status & IRQ_DISABLED) + return 0; + + cd.p = data->chip_data; + mask = 1ull << cd.s.bit; + + if (cd.s.line == 0) { + for_each_online_cpu(cpu) { + unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu); + int index = octeon_coreid_for_cpu(cpu) * 2; + if (cpumask_test_cpu(cpu, dest) && enable_one) { + enable_one = 0; + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask); + } else { + clear_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask); + } + } + } else { + for_each_online_cpu(cpu) { + unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); + int index = octeon_coreid_for_cpu(cpu) * 2 + 1; + if (cpumask_test_cpu(cpu, dest) && enable_one) { + enable_one = 0; + set_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask); + } else { + clear_bit(cd.s.bit, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask); + } } } return 0; @@ -389,80 +674,101 @@ static int octeon_irq_ciu0_set_affinity_v2(unsigned int irq, #endif /* + * The v1 CIU code already masks things, so supply a dummy version to + * the core chip code. + */ +static void octeon_irq_dummy_mask(struct irq_data *data) +{ + return; +} + +/* * Newer octeon chips have support for lockless CIU operation. */ -static struct irq_chip octeon_irq_chip_ciu0_v2 = { - .name = "CIU0", - .enable = octeon_irq_ciu0_enable_v2, - .disable = octeon_irq_ciu0_disable_all_v2, - .eoi = octeon_irq_ciu0_enable_v2, +static struct irq_chip octeon_irq_chip_ciu_v2 = { + .name = "CIU", + .irq_enable = octeon_irq_ciu_enable_v2, + .irq_disable = octeon_irq_ciu_disable_all_v2, + .irq_mask = octeon_irq_ciu_disable_local_v2, + .irq_unmask = octeon_irq_ciu_enable_v2, #ifdef CONFIG_SMP - .set_affinity = octeon_irq_ciu0_set_affinity_v2, + .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, + .irq_cpu_offline = octeon_irq_cpu_offline_ciu, #endif }; -static struct irq_chip octeon_irq_chip_ciu0 = { - .name = "CIU0", - .enable = octeon_irq_ciu0_enable, - .disable = octeon_irq_ciu0_disable, - .eoi = octeon_irq_ciu0_eoi, +static struct irq_chip octeon_irq_chip_ciu_edge_v2 = { + .name = "CIU-E", + .irq_enable = octeon_irq_ciu_enable_v2, + .irq_disable = octeon_irq_ciu_disable_all_v2, + .irq_ack = octeon_irq_ciu_ack, + .irq_mask = octeon_irq_ciu_disable_local_v2, + .irq_unmask = octeon_irq_ciu_enable_v2, #ifdef CONFIG_SMP - .set_affinity = octeon_irq_ciu0_set_affinity, + .irq_set_affinity = octeon_irq_ciu_set_affinity_v2, + .irq_cpu_offline = octeon_irq_cpu_offline_ciu, #endif }; -/* The mbox versions don't do any affinity or round-robin. */ -static struct irq_chip octeon_irq_chip_ciu0_mbox_v2 = { - .name = "CIU0-M", - .enable = octeon_irq_ciu0_enable_mbox_v2, - .disable = octeon_irq_ciu0_disable, - .eoi = octeon_irq_ciu0_eoi_mbox_v2, +static struct irq_chip octeon_irq_chip_ciu = { + .name = "CIU", + .irq_enable = octeon_irq_ciu_enable, + .irq_disable = octeon_irq_ciu_disable_all, + .irq_mask = octeon_irq_dummy_mask, +#ifdef CONFIG_SMP + .irq_set_affinity = octeon_irq_ciu_set_affinity, + .irq_cpu_offline = octeon_irq_cpu_offline_ciu, +#endif }; -static struct irq_chip octeon_irq_chip_ciu0_mbox = { - .name = "CIU0-M", - .enable = octeon_irq_ciu0_enable_mbox, - .disable = octeon_irq_ciu0_disable, - .eoi = octeon_irq_ciu0_eoi, +static struct irq_chip octeon_irq_chip_ciu_edge = { + .name = "CIU-E", + .irq_enable = octeon_irq_ciu_enable, + .irq_disable = octeon_irq_ciu_disable_all, + .irq_mask = octeon_irq_dummy_mask, + .irq_ack = octeon_irq_ciu_ack, +#ifdef CONFIG_SMP + .irq_set_affinity = octeon_irq_ciu_set_affinity, + .irq_cpu_offline = octeon_irq_cpu_offline_ciu, +#endif }; -static void octeon_irq_ciu1_ack(unsigned int irq) -{ - /* - * In order to avoid any locking accessing the CIU, we - * acknowledge CIU interrupts by disabling all of them. This - * way we can use a per core register and avoid any out of - * core locking requirements. This has the side affect that - * CIU interrupts can't be processed recursively. We don't - * need to disable IRQs to make these atomic since they are - * already disabled earlier in the low level interrupt code. - */ - clear_c0_status(0x100 << 3); -} +/* The mbox versions don't do any affinity or round-robin. */ +static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = { + .name = "CIU-M", + .irq_enable = octeon_irq_ciu_enable_all_v2, + .irq_disable = octeon_irq_ciu_disable_all_v2, + .irq_ack = octeon_irq_ciu_disable_local_v2, + .irq_eoi = octeon_irq_ciu_enable_local_v2, + + .irq_cpu_online = octeon_irq_cpu_online_mbox_v2, + .irq_cpu_offline = octeon_irq_cpu_offline_mbox_v2, +}; -static void octeon_irq_ciu1_eoi(unsigned int irq) -{ - /* - * Enable all CIU interrupts again. We don't need to disable - * IRQs to make these atomic since they are already disabled - * earlier in the low level interrupt code. - */ - set_c0_status(0x100 << 3); -} +static struct irq_chip octeon_irq_chip_ciu_mbox = { + .name = "CIU-M", + .irq_enable = octeon_irq_ciu_enable_all, + .irq_disable = octeon_irq_ciu_disable_all, -static void octeon_irq_ciu1_enable(unsigned int irq) + .irq_cpu_online = octeon_irq_cpu_online_mbox, + .irq_cpu_offline = octeon_irq_cpu_offline_mbox, +}; + +/* + * Watchdog interrupts are special. They are associated with a single + * core, so we hardwire the affinity to that core. + */ +static void octeon_irq_ciu_wd_enable(struct irq_data *data) { - struct irq_desc *desc = irq_to_desc(irq); - int coreid = next_coreid_for_irq(desc); unsigned long flags; - uint64_t en1; - int bit = irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ + unsigned long *pen; + int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ + int cpu = octeon_cpu_for_coreid(coreid); raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); - en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1)); - en1 |= 1ull << bit; - cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1); - cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1)); + pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); + set_bit(coreid, pen); + cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen); raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); } @@ -470,286 +776,281 @@ static void octeon_irq_ciu1_enable(unsigned int irq) * Watchdog interrupts are special. They are associated with a single * core, so we hardwire the affinity to that core. */ -static void octeon_irq_ciu1_wd_enable(unsigned int irq) +static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data) { - unsigned long flags; - uint64_t en1; - int bit = irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */ - int coreid = bit; + int coreid = data->irq - OCTEON_IRQ_WDOG0; + int cpu = octeon_cpu_for_coreid(coreid); - raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags); - en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1)); - en1 |= 1ull << bit; - cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1); - cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1)); - raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags); + |