From e8c7ef7d5819cb7ece70ce0d22bce63ca45a29e2 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 4 Jun 2020 13:55:14 -0700 Subject: RISC-V: Sort select statements alphanumerically Like patch b1b3f49 ("ARM: config: sort select statements alphanumerically") , we sort all our select statements alphanumerically by using the perl script in patch b1b3f49 as above. As suggested by Andrew Morton: This is a pet peeve of mine. Any time there's a long list of items (header file inclusions, kconfig entries, array initalisers, etc) and someone wants to add a new item, they *always* go and stick it at the end of the list. Guys, don't do this. Either put the new item into a randomly-chosen position or, probably better, alphanumerically sort the list. Suggested-by: Zong Li [Palmer: Re-ran the script, as there were predictably a bunch of conflicts] Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 68 +++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c733007b90ab..8244b8f7e7c3 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,64 +12,64 @@ config 32BIT config RISCV def_bool y - select OF - select OF_EARLY_FLATTREE - select OF_IRQ select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_WX + select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_GIGANTIC_PAGE + select ARCH_HAS_MMIOWB + select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_SET_DIRECT_MAP + select ARCH_HAS_SET_MEMORY + select ARCH_HAS_STRICT_KERNEL_RWX if MMU + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select ARCH_WANT_FRAME_POINTERS + select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select CLONE_BACKWARDS select COMMON_CLK + select EDAC_SUPPORT + select GENERIC_ARCH_TOPOLOGY if SMP + select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS + select GENERIC_IOREMAP + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW select GENERIC_PCI_IOMAP + select GENERIC_PTDUMP if MMU select GENERIC_SCHED_CLOCK + select GENERIC_SMP_IDLE_THREAD select GENERIC_STRNCPY_FROM_USER if MMU select GENERIC_STRNLEN_USER if MMU - select GENERIC_SMP_IDLE_THREAD - select GENERIC_ATOMIC64 if !64BIT - select GENERIC_IOREMAP - select GENERIC_PTDUMP if MMU select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_KASAN if MMU && 64BIT + select HAVE_ARCH_KGDB + select HAVE_ARCH_KGDB_QXFER_PKT + select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TRACEHOOK select HAVE_ASM_MODVERSIONS + select HAVE_COPY_THREAD_TLS select HAVE_DMA_CONTIGUOUS if MMU + select HAVE_EBPF_JIT if MMU select HAVE_FUTEX_CMPXCHG if FUTEX + select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN - select SPARSE_IRQ - select SYSCTL_EXCEPTION_TRACE - select HAVE_ARCH_TRACEHOOK - select HAVE_PCI select MODULES_USE_ELF_RELA if MODULES select MODULE_SECTIONS if MODULES - select THREAD_INFO_IN_TASK + select OF + select OF_EARLY_FLATTREE + select OF_IRQ select PCI_DOMAINS_GENERIC if PCI select PCI_MSI if PCI select RISCV_TIMER - select GENERIC_IRQ_MULTI_HANDLER - select GENERIC_ARCH_TOPOLOGY if SMP - select ARCH_HAS_PTE_SPECIAL - select ARCH_HAS_MMIOWB - select ARCH_HAS_DEBUG_VIRTUAL if MMU - select HAVE_EBPF_JIT if MMU - select EDAC_SUPPORT - select ARCH_HAS_GIGANTIC_PAGE - select ARCH_HAS_SET_DIRECT_MAP - select ARCH_HAS_SET_MEMORY - select ARCH_HAS_STRICT_KERNEL_RWX if MMU - select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select SPARSEMEM_STATIC if 32BIT - select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU - select HAVE_ARCH_MMAP_RND_BITS if MMU - select ARCH_HAS_GCOV_PROFILE_ALL - select HAVE_COPY_THREAD_TLS - select HAVE_ARCH_KASAN if MMU && 64BIT - select HAVE_ARCH_KGDB - select HAVE_ARCH_KGDB_QXFER_PKT + select SPARSE_IRQ + select SYSCTL_EXCEPTION_TRACE + select THREAD_INFO_IN_TASK config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT @@ -196,11 +196,11 @@ config ARCH_RV64I bool "RV64I" select 64BIT select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && GCC_VERSION >= 50000 - select HAVE_FUNCTION_TRACER - select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE if MMU select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE + select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER select SWIOTLB if MMU endchoice -- cgit v1.2.3 From 5cf998ba8c7bbbfec3e0d6534c9a3f860ab9920e Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 14:45:38 +0530 Subject: RISC-V: self-contained IPI handling routine Currently, the IPI handling routine riscv_software_interrupt() does not take any argument and also does not perform irq_enter()/irq_exit(). This patch makes IPI handling routine more self-contained by: 1. Passing "pt_regs *" argument 2. Explicitly doing irq_enter()/irq_exit() 3. Explicitly save/restore "pt_regs *" using set_irq_regs() With above changes, IPI handling routine does not depend on caller function to perform irq_enter()/irq_exit() and save/restore of "pt_regs *" hence its more self-contained. This also enables us to call IPI handling routine from IRQCHIP drivers. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/irq.h | 1 - arch/riscv/include/asm/smp.h | 3 +++ arch/riscv/kernel/irq.c | 16 ++++++++++------ arch/riscv/kernel/smp.c | 11 +++++++++-- 4 files changed, 22 insertions(+), 9 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 6e1b0e0325eb..0183e15ace66 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -13,7 +13,6 @@ #define NR_IRQS 0 void riscv_timer_interrupt(void); -void riscv_software_interrupt(void); #include diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index f4c7cfda6b7f..40bb1c15a731 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -28,6 +28,9 @@ void show_ipi_stats(struct seq_file *p, int prec); /* SMP initialization hook for setup_arch */ void __init setup_smp(void); +/* Called from C code, this handles an IPI. */ +void handle_IPI(struct pt_regs *regs); + /* Hook for the generic smp_call_function_many() routine. */ void arch_send_call_function_ipi_mask(struct cpumask *mask); diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 345c4f2eba13..bb0bfcd537e7 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -19,12 +19,15 @@ int arch_show_interrupts(struct seq_file *p, int prec) asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) { - struct pt_regs *old_regs = set_irq_regs(regs); + struct pt_regs *old_regs; - irq_enter(); switch (regs->cause & ~CAUSE_IRQ_FLAG) { case RV_IRQ_TIMER: + old_regs = set_irq_regs(regs); + irq_enter(); riscv_timer_interrupt(); + irq_exit(); + set_irq_regs(old_regs); break; #ifdef CONFIG_SMP case RV_IRQ_SOFT: @@ -32,19 +35,20 @@ asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) * We only use software interrupts to pass IPIs, so if a non-SMP * system gets one, then we don't know what to do. */ - riscv_software_interrupt(); + handle_IPI(regs); break; #endif case RV_IRQ_EXT: + old_regs = set_irq_regs(regs); + irq_enter(); handle_arch_irq(regs); + irq_exit(); + set_irq_regs(old_regs); break; default: pr_alert("unexpected interrupt cause 0x%lx", regs->cause); BUG(); } - irq_exit(); - - set_irq_regs(old_regs); } void __init init_IRQ(void) diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index a65a8fa0c22d..b1d4f452f843 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -123,11 +123,14 @@ static inline void clear_ipi(void) clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); } -void riscv_software_interrupt(void) +void handle_IPI(struct pt_regs *regs) { + struct pt_regs *old_regs = set_irq_regs(regs); unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; unsigned long *stats = ipi_data[smp_processor_id()].stats; + irq_enter(); + clear_ipi(); while (true) { @@ -138,7 +141,7 @@ void riscv_software_interrupt(void) ops = xchg(pending_ipis, 0); if (ops == 0) - return; + goto done; if (ops & (1 << IPI_RESCHEDULE)) { stats[IPI_RESCHEDULE]++; @@ -160,6 +163,10 @@ void riscv_software_interrupt(void) /* Order data access and bit testing. */ mb(); } + +done: + irq_exit(); + set_irq_regs(old_regs); } static const char * const ipi_names[] = { -- cgit v1.2.3 From d175d699df07041befda3779f29bf7126c298730 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 14:45:39 +0530 Subject: RISC-V: Rename and move plic_find_hart_id() to arch directory The plic_find_hart_id() can be useful to other interrupt controller drivers (such as RISC-V local interrupt driver) so we rename this function to riscv_of_parent_hartid() and place it in arch directory along with riscv_of_processor_hartid(). Signed-off-by: Anup Patel Reviewed-by: Atish Patra Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Signed-off-by: Palmer Dabbelt Reviewed-by: Marc Zyngier Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/processor.h | 1 + arch/riscv/kernel/cpu.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'arch/riscv') diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 3ddb798264f1..b1efd840003c 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -75,6 +75,7 @@ static inline void wait_for_interrupt(void) struct device_node; int riscv_of_processor_hartid(struct device_node *node); +int riscv_of_parent_hartid(struct device_node *node); extern void riscv_fill_hwcap(void); diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 40a3c442ac5f..6d59e6906fdd 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -44,6 +44,22 @@ int riscv_of_processor_hartid(struct device_node *node) return hart; } +/* + * Find hart ID of the CPU DT node under which given DT node falls. + * + * To achieve this, we walk up the DT tree until we find an active + * RISC-V core (HART) node and extract the cpuid from it. + */ +int riscv_of_parent_hartid(struct device_node *node) +{ + for (; node; node = node->parent) { + if (of_device_is_compatible(node, "riscv")) + return riscv_of_processor_hartid(node); + } + + return -1; +} + #ifdef CONFIG_PROC_FS static void print_isa(struct seq_file *f, const char *isa) -- cgit v1.2.3 From 6b7ce8927b5a4d739670d4dc0de301f2abfd9a5c Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 14:45:40 +0530 Subject: irqchip: RISC-V per-HART local interrupt controller driver The RISC-V per-HART local interrupt controller manages software interrupts, timer interrupts, external interrupts (which are routed via the platform level interrupt controller) and other per-HART local interrupts. We add a driver for the RISC-V local interrupt controller, which eventually replaces the RISC-V architecture code, allowing for a better split between arch code and drivers. The driver is compliant with RISC-V Hart-Level Interrupt Controller DT bindings located at: Documentation/devicetree/bindings/interrupt-controller/riscv,cpu-intc.txt Co-developed-by: Palmer Dabbelt Signed-off-by: Anup Patel [Palmer: Cleaned up warnings] Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/irq.h | 2 -- arch/riscv/kernel/irq.c | 33 +++------------------------------ arch/riscv/kernel/traps.c | 2 -- 4 files changed, 4 insertions(+), 34 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 8244b8f7e7c3..4f5bff0e7ab2 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -40,6 +40,7 @@ config RISCV select GENERIC_SMP_IDLE_THREAD select GENERIC_STRNCPY_FROM_USER if MMU select GENERIC_STRNLEN_USER if MMU + select HANDLE_DOMAIN_IRQ select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_KASAN if MMU && 64BIT select HAVE_ARCH_KGDB diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 0183e15ace66..a9e5f07a7e9c 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -10,8 +10,6 @@ #include #include -#define NR_IRQS 0 - void riscv_timer_interrupt(void); #include diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index bb0bfcd537e7..eb8777642ce6 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -19,39 +18,13 @@ int arch_show_interrupts(struct seq_file *p, int prec) asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) { - struct pt_regs *old_regs; - - switch (regs->cause & ~CAUSE_IRQ_FLAG) { - case RV_IRQ_TIMER: - old_regs = set_irq_regs(regs); - irq_enter(); - riscv_timer_interrupt(); - irq_exit(); - set_irq_regs(old_regs); - break; -#ifdef CONFIG_SMP - case RV_IRQ_SOFT: - /* - * We only use software interrupts to pass IPIs, so if a non-SMP - * system gets one, then we don't know what to do. - */ - handle_IPI(regs); - break; -#endif - case RV_IRQ_EXT: - old_regs = set_irq_regs(regs); - irq_enter(); + if (handle_arch_irq) handle_arch_irq(regs); - irq_exit(); - set_irq_regs(old_regs); - break; - default: - pr_alert("unexpected interrupt cause 0x%lx", regs->cause); - BUG(); - } } void __init init_IRQ(void) { irqchip_init(); + if (!handle_arch_irq) + panic("No interrupt controller found."); } diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 5080fdf8c296..ecec1778e3a4 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -183,6 +183,4 @@ void trap_init(void) csr_write(CSR_SCRATCH, 0); /* Set the exception vector address */ csr_write(CSR_TVEC, &handle_exception); - /* Enable interrupts */ - csr_write(CSR_IE, IE_SIE); } -- cgit v1.2.3 From 033a65de7eced89f62d2cd166b1ee2d33af4f1e4 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 14:45:41 +0530 Subject: clocksource/drivers/timer-riscv: Use per-CPU timer interrupt Instead of directly calling RISC-V timer interrupt handler from RISC-V local interrupt conntroller driver, this patch implements RISC-V timer interrupt as a per-CPU interrupt using per-CPU APIs of Linux IRQ subsystem. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Reviewed-by: Marc Zyngier Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/irq.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index a9e5f07a7e9c..9807ad164015 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -10,8 +10,6 @@ #include #include -void riscv_timer_interrupt(void); - #include #endif /* _ASM_RISCV_IRQ_H */ -- cgit v1.2.3 From 24dc17005ca1d5034d5575f8f94bfd9ad719a22f Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 14:45:42 +0530 Subject: RISC-V: Remove do_IRQ() function The only thing do_IRQ() does is call handle_arch_irq function pointer. We can very well call handle_arch_irq function pointer directly from assembly and remove do_IRQ() function hence this patch. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/entry.S | 4 +++- arch/riscv/kernel/irq.c | 6 ------ 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 56d071b2c0a1..cae7e6d4c7ef 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -106,7 +106,9 @@ _save_context: /* Handle interrupts */ move a0, sp /* pt_regs */ - tail do_IRQ + la a1, handle_arch_irq + REG_L a1, (a1) + jr a1 1: /* * Exceptions run with interrupts enabled or disabled depending on the diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index eb8777642ce6..7207fa08d78f 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -16,12 +16,6 @@ int arch_show_interrupts(struct seq_file *p, int prec) return 0; } -asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) -{ - if (handle_arch_irq) - handle_arch_irq(regs); -} - void __init init_IRQ(void) { irqchip_init(); -- cgit v1.2.3 From e71ee06e3ca337758ac95288a917622ff9808fa0 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 14:45:43 +0530 Subject: RISC-V: Force select RISCV_INTC for CONFIG_RISCV The RISC-V per-HART local interrupt controller driver is mandatory for all RISC-V system (with/without MMU) hence we force select it for CONFIG_RISCV (just like RISCV_TIMER). Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/riscv') diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 4f5bff0e7ab2..7c9614909d2c 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -66,6 +66,7 @@ config RISCV select OF_IRQ select PCI_DOMAINS_GENERIC if PCI select PCI_MSI if PCI + select RISCV_INTC select RISCV_TIMER select SPARSEMEM_STATIC if 32BIT select SPARSE_IRQ -- cgit v1.2.3 From 4e0f9e3a6104261f25b16fcab02fc96f5666ba11 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 1 Jun 2020 10:36:56 +0530 Subject: RISC-V: Don't mark init section as non-executable The head text section (i.e. _start, secondary_start_sbi, etc) and the init section fall under same page table level-1 mapping. Currently, the runtime CPU hotplug is broken because we are marking init section as non-executable which in-turn marks head text section as non-executable. Further investigating other architectures, it seems marking the init section as non-executable is redundant because the init section pages are anyway poisoned and freed. To fix broken runtime CPU hotplug, we simply remove the code marking the init section as non-executable. Fixes: d27c3c90817e ("riscv: add STRICT_KERNEL_RWX support") Cc: stable@vger.kernel.org Signed-off-by: Anup Patel Reviewed-by: Zong Li Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/init.c | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 34327407b0c5..6b2fb2ca0302 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -481,17 +481,6 @@ static void __init setup_vm_final(void) csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); local_flush_tlb_all(); } - -void free_initmem(void) -{ - unsigned long init_begin = (unsigned long)__init_begin; - unsigned long init_end = (unsigned long)__init_end; - - /* Make the region as non-execuatble. */ - set_memory_nx(init_begin, (init_end - init_begin) >> PAGE_SHIFT); - free_initmem_default(POISON_FREE_INITMEM); -} - #else asmlinkage void __init setup_vm(uintptr_t dtb_pa) { -- cgit v1.2.3 From 05589dde649cd954a122b0ebb2cbe4ade8a80e98 Mon Sep 17 00:00:00 2001 From: Zong Li Date: Mon, 1 Jun 2020 15:10:58 +0800 Subject: riscv: fix build warning of missing prototypes Add the missing header in file, it was lost in original implementation. The warning message as follows: - no previous prototype for 'patch_text_nosync' [-Wmissing-prototypes] - no previous prototype for 'patch_text' [-Wmissing-prototypes] Changed in v2: - Correct the typo of commit message. Signed-off-by: Zong Li Reported-by: kbuild test robot Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/patch.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/riscv') diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 5805791cd5b5..d4a64dfed342 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -11,6 +11,7 @@ #include #include #include +#include struct patch_insn { void *addr; -- cgit v1.2.3 From ad5d1122b82fbd6a816d1b9d26ee01a6dbc2d757 Mon Sep 17 00:00:00 2001 From: Vincent Chen Date: Tue, 9 Jun 2020 22:14:48 +0800 Subject: riscv: use vDSO common flow to reduce the latency of the time-related functions Even if RISC-V has supported the vDSO feature, the latency of the functions for obtaining the system time is still expensive. It is because these functions still trigger a corresponding system call in the process, which slows down the response time. If we want to remove the system call to reduce the latency, the kernel should have the ability to output the system clock information to userspace. This patch introduces the vDSO common flow to enable the kernel to achieve the above feature and uses "rdtime" instruction to obtain the current time in the user space. Under this condition, the latency cost by the ecall from U-mode to S-mode can be eliminated. After applying this patch, the latency of gettimeofday() measured on the HiFive unleashed board can be reduced by %61. Signed-off-by: Vincent Chen Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 4 ++ arch/riscv/include/asm/clocksource.h | 7 +++ arch/riscv/include/asm/processor.h | 12 +---- arch/riscv/include/asm/vdso.h | 2 + arch/riscv/include/asm/vdso/clocksource.h | 8 +++ arch/riscv/include/asm/vdso/gettimeofday.h | 79 ++++++++++++++++++++++++++++++ arch/riscv/include/asm/vdso/processor.h | 19 +++++++ arch/riscv/include/asm/vdso/vsyscall.h | 27 ++++++++++ arch/riscv/kernel/time.c | 9 ++++ arch/riscv/kernel/vdso.c | 8 ++- arch/riscv/kernel/vdso/Makefile | 12 +++-- arch/riscv/kernel/vdso/clock_getres.S | 18 ------- arch/riscv/kernel/vdso/clock_gettime.S | 18 ------- arch/riscv/kernel/vdso/gettimeofday.S | 18 ------- arch/riscv/kernel/vdso/vdso.lds.S | 2 + arch/riscv/kernel/vdso/vgettimeofday.c | 25 ++++++++++ 16 files changed, 199 insertions(+), 69 deletions(-) create mode 100644 arch/riscv/include/asm/clocksource.h create mode 100644 arch/riscv/include/asm/vdso/clocksource.h create mode 100644 arch/riscv/include/asm/vdso/gettimeofday.h create mode 100644 arch/riscv/include/asm/vdso/processor.h create mode 100644 arch/riscv/include/asm/vdso/vsyscall.h delete mode 100644 arch/riscv/kernel/vdso/clock_getres.S delete mode 100644 arch/riscv/kernel/vdso/clock_gettime.S delete mode 100644 arch/riscv/kernel/vdso/gettimeofday.S create mode 100644 arch/riscv/kernel/vdso/vgettimeofday.c (limited to 'arch/riscv') diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7c9614909d2c..128192e14ff2 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,6 +12,7 @@ config 32BIT config RISCV def_bool y + select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_WX @@ -31,6 +32,7 @@ config RISCV select GENERIC_ARCH_TOPOLOGY if SMP select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS + select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO select GENERIC_IOREMAP select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW @@ -40,6 +42,7 @@ config RISCV select GENERIC_SMP_IDLE_THREAD select GENERIC_STRNCPY_FROM_USER if MMU select GENERIC_STRNLEN_USER if MMU + select GENERIC_TIME_VSYSCALL if MMU && 64BIT select HANDLE_DOMAIN_IRQ select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_KASAN if MMU && 64BIT @@ -53,6 +56,7 @@ config RISCV select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU select HAVE_FUTEX_CMPXCHG if FUTEX + select HAVE_GENERIC_VDSO if MMU && 64BIT select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS diff --git a/arch/riscv/include/asm/clocksource.h b/arch/riscv/include/asm/clocksource.h new file mode 100644 index 000000000000..482185566b0c --- /dev/null +++ b/arch/riscv/include/asm/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_CLOCKSOURCE_H +#define _ASM_CLOCKSOURCE_H + +#include + +#endif diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index b1efd840003c..bdddcd5c1b71 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -8,6 +8,8 @@ #include +#include + #include /* @@ -58,16 +60,6 @@ static inline void release_thread(struct task_struct *dead_task) extern unsigned long get_wchan(struct task_struct *p); -static inline void cpu_relax(void) -{ -#ifdef __riscv_muldiv - int dummy; - /* In lieu of a halt instruction, induce a long-latency stall. */ - __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); -#endif - barrier(); -} - static inline void wait_for_interrupt(void) { __asm__ __volatile__ ("wfi"); diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index 7a7fce63c474..8454f746bbfd 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -10,8 +10,10 @@ #include +#ifndef GENERIC_TIME_VSYSCALL struct vdso_data { }; +#endif /* * The VDSO symbols are mapped into Linux so we can just use regular symbol diff --git a/arch/riscv/include/asm/vdso/clocksource.h b/arch/riscv/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..df6ea65c1dec --- /dev/null +++ b/arch/riscv/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..c8e818688ec1 --- /dev/null +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_GETTIMEOFDAY_H +#define __ASM_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm("a7") = __NR_gettimeofday; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_gettime; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_getres; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) +{ + /* + * The purpose of csr_read(CSR_TIME) is to trap the system into + * M-mode to obtain the value of CSR_TIME. Hence, unlike other + * architecture, no fence instructions surround the csr_read() + */ + return csr_read(CSR_TIME); +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h new file mode 100644 index 000000000000..82a5693b1861 --- /dev/null +++ b/arch/riscv/include/asm/vdso/processor.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +static inline void cpu_relax(void) +{ +#ifdef __riscv_muldiv + int dummy; + /* In lieu of a halt instruction, induce a long-latency stall. */ + __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); +#endif + barrier(); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/vdso/vsyscall.h b/arch/riscv/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..82fd5d83bd60 --- /dev/null +++ b/arch/riscv/include/asm/vdso/vsyscall.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_VSYSCALL_H +#define __ASM_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include +#include + +extern struct vdso_data *vdso_data; + +/* + * Update the vDSO data page to keep in sync with kernel timekeeping. + */ +static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void) +{ + return vdso_data; +} + +#define __arch_get_k_vdso_data __riscv_get_k_vdso_data + +/* The asm-generic header needs to be included after the definitions above */ +#include + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 6a53c02e9c73..4d3a1048ad8b 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -26,3 +26,12 @@ void __init time_init(void) lpj_fine = riscv_timebase / HZ; timer_probe(); } + +void clocksource_arch_init(struct clocksource *cs) +{ +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER; +#else + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; +#endif +} diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 484d95a70907..70b6461a6387 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -11,8 +11,12 @@ #include #include #include - +#include +#ifdef GENERIC_TIME_VSYSCALL +#include +#else #include +#endif extern char vdso_start[], vdso_end[]; @@ -26,7 +30,7 @@ static union { struct vdso_data data; u8 page[PAGE_SIZE]; } vdso_data_store __page_aligned_data; -static struct vdso_data *vdso_data = &vdso_data_store.data; +struct vdso_data *vdso_data = &vdso_data_store.data; static int __init vdso_init(void) { diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 4c8b2a4a6a70..38ba55b0eb9d 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -1,12 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-only # Copied from arch/tile/kernel/vdso/Makefile +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_RISCV_32|R_RISCV_64|R_RISCV_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile # Symbols present in the vdso vdso-syms = rt_sigreturn ifdef CONFIG_64BIT -vdso-syms += gettimeofday -vdso-syms += clock_gettime -vdso-syms += clock_getres +vdso-syms += vgettimeofday endif vdso-syms += getcpu vdso-syms += flush_icache @@ -14,6 +16,10 @@ vdso-syms += flush_icache # Files to link into the vdso obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + # Build rules targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) diff --git a/arch/riscv/kernel/vdso/clock_getres.S b/arch/riscv/kernel/vdso/clock_getres.S deleted file mode 100644 index 91378a52eb22..000000000000 --- a/arch/riscv/kernel/vdso/clock_getres.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include -#include - - .text -/* int __vdso_clock_getres(clockid_t clock_id, struct timespec *res); */ -ENTRY(__vdso_clock_getres) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_clock_getres - ecall - ret - .cfi_endproc -ENDPROC(__vdso_clock_getres) diff --git a/arch/riscv/kernel/vdso/clock_gettime.S b/arch/riscv/kernel/vdso/clock_gettime.S deleted file mode 100644 index 5371fd9bc01f..000000000000 --- a/arch/riscv/kernel/vdso/clock_gettime.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include -#include - - .text -/* int __vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); */ -ENTRY(__vdso_clock_gettime) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_clock_gettime - ecall - ret - .cfi_endproc -ENDPROC(__vdso_clock_gettime) diff --git a/arch/riscv/kernel/vdso/gettimeofday.S b/arch/riscv/kernel/vdso/gettimeofday.S deleted file mode 100644 index e6fb8af88632..000000000000 --- a/arch/riscv/kernel/vdso/gettimeofday.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include -#include - - .text -/* int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); */ -ENTRY(__vdso_gettimeofday) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_gettimeofday - ecall - ret - .cfi_endproc -ENDPROC(__vdso_gettimeofday) diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index f66a091cb890..e6f558bca71b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -2,11 +2,13 @@ /* * Copyright (C) 2012 Regents of the University of California */ +#include OUTPUT_ARCH(riscv) SECTIONS { + PROVIDE(_vdso_data = . + PAGE_SIZE); . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/riscv/kernel/vdso/vgettimeofday.c b/arch/riscv/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000000..d264943e2e47 --- /dev/null +++ b/arch/riscv/kernel/vdso/vgettimeofday.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from arch/arm64/kernel/vdso/vgettimeofday.c + * + * Copyright (C) 2018 ARM Ltd. + * Copyright (C) 2020 SiFive + */ + +#include +#include + +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); +} -- cgit v1.2.3 From 01f76386b0ac0b1c1094456c8f43ea6085fc49d2 Mon Sep 17 00:00:00 2001 From: Vincent Chen Date: Tue, 9 Jun 2020 22:14:49 +0800 Subject: riscv: set the permission of vdso_data to read-only The original vdso_data page is empty, so the permission of the vdso_data page can be the same with the vdso text page. After introducing the vDSO common flow, the vdso_data is not empty and the permission should be changed to read-only. Signed-off-by: Vincent Chen Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vdso.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'arch/riscv') diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 70b6461a6387..94450332aa4f 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -79,13 +79,22 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, */ mm->context.vdso = (void *)vdso_base; - ret = install_special_mapping(mm, vdso_base, vdso_len, + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), vdso_pagelist); - if (unlikely(ret)) + if (unlikely(ret)) { mm->context.vdso = NULL; + goto end; + } + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); + + if (unlikely(ret)) + mm->context.vdso = NULL; end: up_write(&mm->mmap_sem); return ret; @@ -95,5 +104,8 @@ const char *arch_vma_name(struct vm_area_struct *vma) { if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) return "[vdso]"; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; return NULL; } -- cgit v1.2.3