diff options
Diffstat (limited to 'arch/arm/mm')
-rw-r--r-- | arch/arm/mm/Kconfig | 90 | ||||
-rw-r--r-- | arch/arm/mm/Makefile | 1 | ||||
-rw-r--r-- | arch/arm/mm/alignment.c | 2 | ||||
-rw-r--r-- | arch/arm/mm/cache-l2x0.c | 33 | ||||
-rw-r--r-- | arch/arm/mm/cache-uniphier.c | 544 | ||||
-rw-r--r-- | arch/arm/mm/context.c | 38 | ||||
-rw-r--r-- | arch/arm/mm/dma-mapping.c | 270 | ||||
-rw-r--r-- | arch/arm/mm/fault.c | 22 | ||||
-rw-r--r-- | arch/arm/mm/fault.h | 1 | ||||
-rw-r--r-- | arch/arm/mm/flush.c | 17 | ||||
-rw-r--r-- | arch/arm/mm/highmem.c | 10 | ||||
-rw-r--r-- | arch/arm/mm/idmap.c | 4 | ||||
-rw-r--r-- | arch/arm/mm/init.c | 117 | ||||
-rw-r--r-- | arch/arm/mm/ioremap.c | 9 | ||||
-rw-r--r-- | arch/arm/mm/mmap.c | 3 | ||||
-rw-r--r-- | arch/arm/mm/mmu.c | 144 | ||||
-rw-r--r-- | arch/arm/mm/pageattr.c | 3 | ||||
-rw-r--r-- | arch/arm/mm/proc-mohawk.S | 2 | ||||
-rw-r--r-- | arch/arm/mm/proc-v7.S | 27 | ||||
-rw-r--r-- | arch/arm/mm/proc-v7m.S | 14 |
20 files changed, 1095 insertions, 256 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index df7537f12469..55347662e5ed 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -21,7 +21,7 @@ config CPU_ARM7TDMI # ARM720T config CPU_ARM720T - bool "Support ARM720T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) + bool select CPU_32v4T select CPU_ABRT_LV4T select CPU_CACHE_V4 @@ -39,7 +39,7 @@ config CPU_ARM720T # ARM740T config CPU_ARM740T - bool "Support ARM740T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) + bool depends on !MMU select CPU_32v4T select CPU_ABRT_LV4T @@ -71,7 +71,7 @@ config CPU_ARM9TDMI # ARM920T config CPU_ARM920T - bool "Support ARM920T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) + bool select CPU_32v4T select CPU_ABRT_EV4T select CPU_CACHE_V4WT @@ -89,7 +89,7 @@ config CPU_ARM920T # ARM922T config CPU_ARM922T - bool "Support ARM922T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) + bool select CPU_32v4T select CPU_ABRT_EV4T select CPU_CACHE_V4WT @@ -108,7 +108,7 @@ config CPU_ARM922T # ARM925T config CPU_ARM925T - bool "Support ARM925T processor" if ARCH_OMAP1 + bool select CPU_32v4T select CPU_ABRT_EV4T select CPU_CACHE_V4WT @@ -127,7 +127,7 @@ config CPU_ARM925T # ARM926T config CPU_ARM926T - bool "Support ARM926T processor" if (!ARCH_MULTIPLATFORM || ARCH_MULTI_V5) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB) + bool select CPU_32v5 select CPU_ABRT_EV5TJ select CPU_CACHE_VIVT @@ -163,7 +163,7 @@ config CPU_FA526 # ARM940T config CPU_ARM940T - bool "Support ARM940T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) + bool depends on !MMU select CPU_32v4T select CPU_ABRT_NOMMU @@ -181,7 +181,7 @@ config CPU_ARM940T # ARM946E-S config CPU_ARM946E - bool "Support ARM946E-S processor" if (ARCH_MULTI_V5 && ARCH_INTEGRATOR) + bool depends on !MMU select CPU_32v5 select CPU_ABRT_NOMMU @@ -198,7 +198,7 @@ config CPU_ARM946E # ARM1020 - needs validating config CPU_ARM1020 - bool "Support ARM1020T (rev 0) processor" if (ARCH_MULTI_V5 && ARCH_INTEGRATOR) + bool select CPU_32v5 select CPU_ABRT_EV4T select CPU_CACHE_V4WT @@ -216,7 +216,7 @@ config CPU_ARM1020 # ARM1020E - needs validating config CPU_ARM1020E - bool "Support ARM1020E processor" if (ARCH_MULTI_V5 && ARCH_INTEGRATOR) + bool depends on n select CPU_32v5 select CPU_ABRT_EV4T @@ -229,7 +229,7 @@ config CPU_ARM1020E # ARM1022E config CPU_ARM1022 - bool "Support ARM1022E processor" if (ARCH_MULTI_V5 && ARCH_INTEGRATOR) + bool select CPU_32v5 select CPU_ABRT_EV4T select CPU_CACHE_VIVT @@ -247,7 +247,7 @@ config CPU_ARM1022 # ARM1026EJ-S config CPU_ARM1026 - bool "Support ARM1026EJ-S processor" if (ARCH_MULTI_V5 && ARCH_INTEGRATOR) + bool select CPU_32v5 select CPU_ABRT_EV5T # But need Jazelle, but EV5TJ ignores bit 10 select CPU_CACHE_VIVT @@ -358,7 +358,7 @@ config CPU_PJ4B # ARMv6 config CPU_V6 - bool "Support ARM V6 processor" if (!ARCH_MULTIPLATFORM || ARCH_MULTI_V6) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX) + bool select CPU_32v6 select CPU_ABRT_EV6 select CPU_CACHE_V6 @@ -371,7 +371,7 @@ config CPU_V6 # ARMv6k config CPU_V6K - bool "Support ARM V6K processor" if (!ARCH_MULTIPLATFORM || ARCH_MULTI_V6) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX) + bool select CPU_32v6 select CPU_32v6K select CPU_ABRT_EV6 @@ -385,7 +385,7 @@ config CPU_V6K # ARMv7 config CPU_V7 - bool "Support ARM V7 processor" if (!ARCH_MULTIPLATFORM || ARCH_MULTI_V7) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX) + bool select CPU_32v6K select CPU_32v7 select CPU_ABRT_EV7 @@ -419,28 +419,24 @@ config CPU_THUMBONLY config CPU_32v3 bool select CPU_USE_DOMAINS if MMU - select NEEDS_SYSCALL_FOR_CMPXCHG if SMP select NEED_KUSER_HELPERS select TLS_REG_EMUL if SMP || !MMU config CPU_32v4 bool select CPU_USE_DOMAINS if MMU - select NEEDS_SYSCALL_FOR_CMPXCHG if SMP select NEED_KUSER_HELPERS select TLS_REG_EMUL if SMP || !MMU config CPU_32v4T bool select CPU_USE_DOMAINS if MMU - select NEEDS_SYSCALL_FOR_CMPXCHG if SMP select NEED_KUSER_HELPERS select TLS_REG_EMUL if SMP || !MMU config CPU_32v5 bool select CPU_USE_DOMAINS if MMU - select NEEDS_SYSCALL_FOR_CMPXCHG if SMP select NEED_KUSER_HELPERS select TLS_REG_EMUL if SMP || !MMU @@ -805,14 +801,6 @@ config TLS_REG_EMUL a few prototypes like that in existence) and therefore access to that required register must be emulated. -config NEEDS_SYSCALL_FOR_CMPXCHG - bool - select NEED_KUSER_HELPERS - help - SMP on a pre-ARMv6 processor? Well OK then. - Forget about fast user space cmpxchg support. - It is just not possible. - config NEED_KUSER_HELPERS bool @@ -986,6 +974,16 @@ config CACHE_TAUROS2 This option enables the Tauros2 L2 cache controller (as found on PJ1/PJ4). +config CACHE_UNIPHIER + bool "Enable the UniPhier outer cache controller" + depends on ARCH_UNIPHIER + default y + select OUTER_CACHE + select OUTER_CACHE_SYNC + help + This option enables the UniPhier outer cache (system cache) + controller. + config CACHE_XSC3L2 bool "Enable the L2 cache on XScale3" depends on CPU_XSC3 @@ -1007,8 +1005,6 @@ config ARM_L1_CACHE_SHIFT config ARM_DMA_MEM_BUFFERABLE bool "Use non-cacheable memory for DMA" if (CPU_V6 || CPU_V6K) && !CPU_V7 - depends on !(MACH_REALVIEW_PB1176 || REALVIEW_EB_ARM11MP || \ - MACH_REALVIEW_PB11MP) default y if CPU_V6 || CPU_V6K || CPU_V7 help Historically, the kernel has used strongly ordered mappings to @@ -1041,24 +1037,26 @@ config ARCH_SUPPORTS_BIG_ENDIAN This option specifies the architecture can support big endian operation. -config ARM_KERNMEM_PERMS - bool "Restrict kernel memory permissions" - depends on MMU - help - If this is set, kernel memory other than kernel text (and rodata) - will be made non-executable. The tradeoff is that each region is - padded to section-size (1MiB) boundaries (because their permissions - are different and splitting the 1M pages into 4K ones causes TLB - performance problems), wasting memory. - config DEBUG_RODATA bool "Make kernel text and rodata read-only" - depends on ARM_KERNMEM_PERMS + depends on MMU && !XIP_KERNEL + default y if CPU_V7 + help + If this is set, kernel text and rodata memory will be made + read-only, and non-text kernel memory will be made non-executable. + The tradeoff is that each region is padded to section-size (1MiB) + boundaries (because their permissions are different and splitting + the 1M pages into 4K ones causes TLB performance problems), which + can waste memory. + +config DEBUG_ALIGN_RODATA + bool "Make rodata strictly non-executable" + depends on DEBUG_RODATA default y help - If this is set, kernel text and rodata will be made read-only. This - is to help catch accidental or malicious attempts to change the - kernel's executable code. Additionally splits rodata from kernel - text so it can be made explicitly non-executable. This creates - another section-size padded region, so it can waste more memory - space while gaining the read-only protections. + If this is set, rodata will be made explicitly non-executable. This + provides protection on the rare chance that attackers might find and + use ROP gadgets that exist in the rodata section. This adds an + additional section-aligned split of rodata from kernel text so it + can be made explicitly non-executable. This padding may waste memory + space to gain the additional protection. diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 57c8df500e8c..7f76d96ce546 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -103,3 +103,4 @@ obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o l2c-l2x0-resume.o obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o +obj-$(CONFIG_CACHE_UNIPHIER) += cache-uniphier.o diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 00b7f7de28a1..7d5f4c736a16 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -803,7 +803,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) } } } else { - fault = probe_kernel_address(instrptr, instr); + fault = probe_kernel_address((void *)instrptr, instr); instr = __mem_to_opcode_arm(instr); } diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 493692d838c6..9f9d54271aad 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -790,7 +790,7 @@ static const struct l2c_init_data l2c310_init_fns __initconst = { }; static int __init __l2c_init(const struct l2c_init_data *data, - u32 aux_val, u32 aux_mask, u32 cache_id) + u32 aux_val, u32 aux_mask, u32 cache_id, bool nosync) { struct outer_cache_fns fns; unsigned way_size_bits, ways; @@ -866,6 +866,10 @@ static int __init __l2c_init(const struct l2c_init_data *data, fns.configure = outer_cache.configure; if (data->fixup) data->fixup(l2x0_base, cache_id, &fns); + if (nosync) { + pr_info("L2C: disabling outer sync\n"); + fns.sync = NULL; + } /* * Check if l2x0 controller is already enabled. If we are booting @@ -925,7 +929,7 @@ void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask) if (data->save) data->save(l2x0_base); - __l2c_init(data, aux_val, aux_mask, cache_id); + __l2c_init(data, aux_val, aux_mask, cache_id, false); } #ifdef CONFIG_OF @@ -1060,6 +1064,18 @@ static void __init l2x0_of_parse(const struct device_node *np, val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT; } + if (of_property_read_bool(np, "arm,parity-enable")) { + mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; + val |= L2C_AUX_CTRL_PARITY_ENABLE; + } else if (of_property_read_bool(np, "arm,parity-disable")) { + mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; + } + + if (of_property_read_bool(np, "arm,shared-override")) { + mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; + val |= L2C_AUX_CTRL_SHARED_OVERRIDE; + } + ret = l2x0_cache_size_of_parse(np, aux_val, aux_mask, &assoc, SZ_256K); if (ret) return; @@ -1176,6 +1192,14 @@ static void __init l2c310_of_parse(const struct device_node *np, *aux_mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; } + if (of_property_read_bool(np, "arm,parity-enable")) { + *aux_val |= L2C_AUX_CTRL_PARITY_ENABLE; + *aux_mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; + } else if (of_property_read_bool(np, "arm,parity-disable")) { + *aux_val &= ~L2C_AUX_CTRL_PARITY_ENABLE; + *aux_mask &= ~L2C_AUX_CTRL_PARITY_ENABLE; + } + prefetch = l2x0_saved_regs.prefetch_ctrl; ret = of_property_read_u32(np, "arm,double-linefill", &val); @@ -1704,6 +1728,7 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask) struct resource res; u32 cache_id, old_aux; u32 cache_level = 2; + bool nosync = false; np = of_find_matching_node(NULL, l2x0_ids); if (!np) @@ -1742,6 +1767,8 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask) if (cache_level != 2) pr_err("L2C: device tree specifies invalid cache level\n"); + nosync = of_property_read_bool(np, "arm,outer-sync-disable"); + /* Read back current (default) hardware configuration */ if (data->save) data->save(l2x0_base); @@ -1756,6 +1783,6 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask) else cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID); - return __l2c_init(data, aux_val, aux_mask, cache_id); + return __l2c_init(data, aux_val, aux_mask, cache_id, nosync); } #endif diff --git a/arch/arm/mm/cache-uniphier.c b/arch/arm/mm/cache-uniphier.c new file mode 100644 index 000000000000..a6fa7b73fbe0 --- /dev/null +++ b/arch/arm/mm/cache-uniphier.c @@ -0,0 +1,544 @@ +/* + * Copyright (C) 2015 Masahiro Yamada <yamada.masahiro@socionext.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#define pr_fmt(fmt) "uniphier: " fmt + +#include <linux/init.h> +#include <linux/io.h> +#include <linux/log2.h> +#include <linux/of_address.h> +#include <linux/slab.h> +#include <asm/hardware/cache-uniphier.h> +#include <asm/outercache.h> + +/* control registers */ +#define UNIPHIER_SSCC 0x0 /* Control Register */ +#define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */ +#define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */ +#define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */ +#define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */ +#define UNIPHIER_SSCC_ON BIT(0) /* enable cache */ +#define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */ +#define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */ + +/* revision registers */ +#define UNIPHIER_SSCID 0x0 /* ID Register */ + +/* operation registers */ +#define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */ +#define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */ +#define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */ +#define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */ +#define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */ +#define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */ +#define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */ +#define UNIPHIER_SSCOQM_TID_MASK (0x3 << 21) +#define UNIPHIER_SSCOQM_TID_LRU_DATA (0x0 << 21) +#define UNIPHIER_SSCOQM_TID_LRU_INST (0x1 << 21) +#define UNIPHIER_SSCOQM_TID_WAY (0x2 << 21) +#define UNIPHIER_SSCOQM_S_MASK (0x3 << 17) +#define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17) +#define UNIPHIER_SSCOQM_S_ALL (0x1 << 17) +#define UNIPHIER_SSCOQM_S_WAY (0x2 << 17) +#define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */ +#define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */ +#define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */ +#define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */ +#define UNIPHIER_SSCOQM_CM_PREFETCH 0x3 /* prefetch to cache */ +#define UNIPHIER_SSCOQM_CM_PREFETCH_BUF 0x4 /* prefetch to pf-buf */ +#define UNIPHIER_SSCOQM_CM_TOUCH 0x5 /* touch */ +#define UNIPHIER_SSCOQM_CM_TOUCH_ZERO 0x6 /* touch to zero */ +#define UNIPHIER_SSCOQM_CM_TOUCH_DIRTY 0x7 /* touch with dirty */ +#define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */ +#define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */ +#define UNIPHIER_SSCOQMASK 0x254 /* Cache Operation Queue Address Mask */ +#define UNIPHIER_SSCOQWN 0x258 /* Cache Operation Queue Way Number */ +#define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/ +#define UNIPHIER_SSCOPPQSEF_FE BIT(1) +#define UNIPHIER_SSCOPPQSEF_OE BIT(0) +#define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */ +#define UNIPHIER_SSCOLPQS_EF BIT(2) +#define UNIPHIER_SSCOLPQS_EST BIT(1) +#define UNIPHIER_SSCOLPQS_QST BIT(0) + +/* Is the touch/pre-fetch destination specified by ways? */ +#define UNIPHIER_SSCOQM_TID_IS_WAY(op) \ + ((op & UNIPHIER_SSCOQM_TID_MASK) == UNIPHIER_SSCOQM_TID_WAY) +/* Is the operation region specified by address range? */ +#define UNIPHIER_SSCOQM_S_IS_RANGE(op) \ + ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE) + +/** + * uniphier_cache_data - UniPhier outer cache specific data + * + * @ctrl_base: virtual base address of control registers + * @rev_base: virtual base address of revision registers + * @op_base: virtual base address of operation registers + * @way_present_mask: each bit specifies if the way is present + * @way_locked_mask: each bit specifies if the way is locked + * @nsets: number of associativity sets + * @line_size: line size in bytes + * @range_op_max_size: max size that can be handled by a single range operation + * @list: list node to include this level in the whole cache hierarchy + */ +struct uniphier_cache_data { + void __iomem *ctrl_base; + void __iomem *rev_base; + void __iomem *op_base; + u32 way_present_mask; + u32 way_locked_mask; + u32 nsets; + u32 line_size; + u32 range_op_max_size; + struct list_head list; +}; + +/* + * List of the whole outer cache hierarchy. This list is only modified during + * the early boot stage, so no mutex is taken for the access to the list. + */ +static LIST_HEAD(uniphier_cache_list); + +/** + * __uniphier_cache_sync - perform a sync point for a particular cache level + * + * @data: cache controller specific data + */ +static void __uniphier_cache_sync(struct uniphier_cache_data *data) +{ + /* This sequence need not be atomic. Do not disable IRQ. */ + writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC, + data->op_base + UNIPHIER_SSCOPE); + /* need a read back to confirm */ + readl_relaxed(data->op_base + UNIPHIER_SSCOPE); +} + +/** + * __uniphier_cache_maint_common - run a queue operation for a particular level + * + * @data: cache controller specific data + * @start: start address of range operation (don't care for "all" operation) + * @size: data size of range operation (don't care for "all" operation) + * @operation: flags to specify the desired cache operation + */ +static void __uniphier_cache_maint_common(struct uniphier_cache_data *data, + unsigned long start, + unsigned long size, + u32 operation) +{ + unsigned long flags; + + /* + * No spin lock is necessary here because: + * + * [1] This outer cache controller is able to accept maintenance + * operations from multiple CPUs at a time in an SMP system; if a + * maintenance operation is under way and another operation is issued, + * the new one is stored in the queue. The controller performs one + * operation after another. If the queue is full, the status register, + * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has + * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have + * different instances for each CPU, i.e. each CPU can track the status + * of the maintenance operations triggered by itself. + * + * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ, + * SSCOQWN}, are shared between multiple CPUs, but the hardware still + * guarantees the registration sequence is atomic; the write access to + * them are arbitrated by the hardware. The first accessor to the + * register, UNIPHIER_SSCOQM, holds the access right and it is released + * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU + * is holding the access right, other CPUs fail to register operations. + * One CPU should not hold the access right for a long time, so local + * IRQs should be disabled while the following sequence. + */ + local_irq_save(flags); + + /* clear the complete notification flag */ + writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS); + + do { + /* set cache operation */ + writel_relaxed(UNIPHIER_SSCOQM_CE | operation, + data->op_base + UNIPHIER_SSCOQM); + + /* set address range if needed */ + if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) { + writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD); + writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ); + } + + /* set target ways if needed */ + if (unlikely(UNIPHIER_SSCOQM_TID_IS_WAY(operation))) + writel_relaxed(data->way_locked_mask, + data->op_base + UNIPHIER_SSCOQWN); + } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) & + (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE))); + + /* wait until the operation is completed */ + while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) != + UNIPHIER_SSCOLPQS_EF)) + cpu_relax(); + + local_irq_restore(flags); +} + +static void __uniphier_cache_maint_all(struct uniphier_cache_data *data, + u32 operation) +{ + __uniphier_cache_maint_common(data, 0, 0, + UNIPHIER_SSCOQM_S_ALL | operation); + + __uniphier_cache_sync(data); +} + +static void __uniphier_cache_maint_range(struct uniphier_cache_data *data, + unsigned long start, unsigned long end, + u32 operation) +{ + unsigned long size; + + /* + * If the start address is not aligned, + * perform a cache operation for the first cache-line + */ + start = start & ~(data->line_size - 1); + + size = end - start; + + if (unlikely(size >= (unsigned long)(-data->line_size))) { + /* this means cache operation for all range */ + __uniphier_cache_maint_all(data, operation); + return; + } + + /* + * If the end address is not aligned, + * perform a cache operation for the last cache-line + */ + size = ALIGN(size, data->line_size); + + while (size) { + unsigned long chunk_size = min_t(unsigned long, size, + data->range_op_max_size); + + __uniphier_cache_maint_common(data, start, chunk_size, + UNIPHIER_SSCOQM_S_RANGE | operation); + + start += chunk_size; + size -= chunk_size; + } + + __uniphier_cache_sync(data); +} + +static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on) +{ + u32 val = 0; + + if (on) + val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON; + + writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC); +} + +static void __init __uniphier_cache_set_locked_ways( + struct uniphier_cache_data *data, + u32 way_mask) +{ + data->way_locked_mask = way_mask & data->way_present_mask; + + writel_relaxed(~data->way_locked_mask & data->way_present_mask, + data->ctrl_base + UNIPHIER_SSCLPDAWCR); +} + +static void uniphier_cache_maint_range(unsigned long start, unsigned long end, + u32 operation) +{ + struct uniphier_cache_data *data; + + list_for_each_entry(data, &uniphier_cache_list, list) + __uniphier_cache_maint_range(data, start, end, operation); +} + +static void uniphier_cache_maint_all(u32 operation) +{ + struct uniphier_cache_data *data; + + list_for_each_entry(data, &uniphier_cache_list, list) + __uniphier_cache_maint_all(data, operation); +} + +static void uniphier_cache_inv_range(unsigned long start, unsigned long end) +{ + uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV); +} + +static void uniphier_cache_clean_range(unsigned long start, unsigned long end) +{ + uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN); +} + +static void uniphier_cache_flush_range(unsigned long start, unsigned long end) +{ + uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH); +} + +static void __init uniphier_cache_inv_all(void) +{ + uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV); +} + +static void uniphier_cache_flush_all(void) +{ + uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH); +} + +static void uniphier_cache_disable(void) +{ + struct uniphier_cache_data *data; + + list_for_each_entry_reverse(data, &uniphier_cache_list, list) + __uniphier_cache_enable(data, false); + + uniphier_cache_flush_all(); +} + +static void __init uniphier_cache_enable(void) +{ + struct uniphier_cache_data *data; + + uniphier_cache_inv_all(); + + list_for_each_entry(data, &uniphier_cache_list, list) { + __uniphier_cache_enable(data, true); + __uniphier_cache_set_locked_ways(data, 0); + } +} + +static void uniphier_cache_sync(void) +{ + struct uniphier_cache_data *data; + + list_for_each_entry(data, &uniphier_cache_list, list) + __uniphier_cache_sync(data); +} + +int __init uniphier_cache_l2_is_enabled(void) +{ + struct uniphier_cache_data *data; + + data = list_first_entry_or_null(&uniphier_cache_list, + struct uniphier_cache_data, list); + if (!data) + return 0; + + return !!(readl_relaxed(data->ctrl_base + UNIPHIER_SSCC) & + UNIPHIER_SSCC_ON); +} + +void __init uniphier_cache_l2_touch_range(unsigned long start, + unsigned long end) +{ + struct uniphier_cache_data *data; + + data = list_first_entry_or_null(&uniphier_cache_list, + struct uniphier_cache_data, list); + if (data) + __uniphier_cache_maint_range(data, start, end, + UNIPHIER_SSCOQM_TID_WAY | + UNIPHIER_SSCOQM_CM_TOUCH); +} + +void __init uniphier_cache_l2_set_locked_ways(u32 way_mask) +{ + struct uniphier_cache_data *data; + + data = list_first_entry_or_null(&uniphier_cache_list, + struct uniphier_cache_data, list); + if (data) + __uniphier_cache_set_locked_ways(data, way_mask); +} + +static const struct of_device_id uniphier_cache_match[] __initconst = { + { + .compatible = "socionext,uniphier-system-cache", + }, + { /* sentinel */ } +}; + +static int __init __uniphier_cache_init(struct device_node *np, + unsigned int *cache_level) +{ + struct uniphier_cache_data *data; + u32 level, cache_size; + struct device_node *next_np; + int ret = 0; + + if (!of_match_node(uniphier_cache_match, np)) { + pr_err("L%d: not compatible with uniphier cache\n", + *cache_level); + return -EINVAL; + } + + if (of_property_read_u32(np, "cache-level", &level)) { + pr_err("L%d: cache-level is not specified\n", *cache_level); + return -EINVAL; + } + + if (level != *cache_level) { + pr_err("L%d: cache-level is unexpected value %d\n", + *cache_level, level); + return -EINVAL; + } + + if (!of_property_read_bool(np, "cache-unified")) { + pr_err("L%d: cache-unified is not specified\n", *cache_level); + return -EINVAL; + } + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (of_property_read_u32(np, "cache-line-size", &data->line_size) || + !is_power_of_2(data->line_size)) { + pr_err("L%d: cache-line-size is unspecified or invalid\n", + *cache_level); + ret = -EINVAL; + goto err; + } + + if (of_property_read_u32(np, "cache-sets", &data->nsets) || + !is_power_of_2(data->nsets)) { + pr_err("L%d: cache-sets is unspecified or invalid\n", + *cache_level); + ret = -EINVAL; + goto err; + } + + if (of_property_read_u32(np, "cache-size", &cache_size) || + cache_size == 0 || cache_size % (data->nsets * data->line_size)) { + pr_err("L%d: cache-size is unspecified or invalid\n", + *cache_level); + ret = -EINVAL; + goto err; + } + + data->way_present_mask = + ((u32)1 << cache_size / data->nsets / data->line_size) - 1; + + data->ctrl_base = of_iomap(np, 0); + if (!data->ctrl_base) { + pr_err("L%d: failed to map control register\n", *cache_level); + ret = -ENOMEM; + goto err; + } + + data->rev_base = of_iomap(np, 1); + if (!data->rev_base) { + pr_err("L%d: failed to map revision register\n", *cache_level); + ret = -ENOMEM; + goto err; + } + + data->op_base = of_iomap(np, 2); + if (!data->op_base) { + pr_err("L%d: failed to map operation register\n", *cache_level); + ret = -ENOMEM; + goto err; + } + + if (*cache_level == 2) { + u32 revision = readl(data->rev_base + UNIPHIER_SSCID); + /* + * The size of range operation is limited to (1 << 22) or less + * for PH-sLD8 or older SoCs. + */ + if (revision <= 0x16) + data->range_op_max_size = (u32)1 << 22; + } + + data->range_op_max_size -= data->line_size; + + INIT_LIST_HEAD(&data->list); + list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */ + + /* + * OK, this level has been successfully initialized. Look for the next + * level cache. Do not roll back even if the initialization of the + * next level cache fails because we want to continue with available + * cache levels. + */ + next_np = of_find_next_cache_node(np); + if (next_np) { + (*cache_level)++; + ret = __uniphier_cache_init(next_np, cache_level); + } + of_node_put(next_np); + + return ret; +err: + iounmap(data->op_base); + iounmap(data->rev_base); + iounmap(data->ctrl_base); + kfree(data); + + return ret; +} + +int __init uniphier_cache_init(void) +{ + struct device_node *np = NULL; + unsigned int cache_level; + int ret = 0; + + /* look for level 2 cache */ + while ((np = of_find_matching_node(np, uniphier_cache_match))) + if (!of_property_read_u32(np, "cache-level", &cache_level) && + cache_level == 2) + break; + + if (!np) + return -ENODEV; + + ret = __uniphier_cache_init(np, &cache_level); + of_node_put(np); + + if (ret) { + /* + * Error out iif L2 initialization fails. Continue with any + * error on L3 or outer because they are optional. + */ + if (cache_level == 2) { + pr_err("failed to initialize L2 cache\n"); + return ret; + } + + cache_level--; + ret = 0; + } + + outer_cache.inv_range = uniphier_cache_inv_range; + outer_cache.clean_range = uniphier_cache_clean_range; + outer_cache.flush_range = uniphier_cache_flush_range; + outer_cache.flush_all = uniphier_cache_flush_all; + outer_cache.disable = uniphier_cache_disable; + outer_cache.sync = uniphier_cache_sync; + + uniphier_cache_enable(); + + pr_info("enabled outer cache (cache level: %d)\n", cache_level); + + return ret; +} diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index 845769e41332..c8c8b9ed02e0 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -165,13 +165,28 @@ static void flush_context(unsigned int cpu) __flush_icache_all(); } -static int is_reserved_asid(u64 asid) +static bool check_update_reserved_asid(u64 asid, u64 newasid) { int cpu; - for_each_possible_cpu(cpu) - if (per_cpu(reserved_asids, cpu) == asid) - return 1; - return 0; + bool hit = false; + + /* |