diff options
50 files changed, 1451 insertions, 29 deletions
diff --git a/Documentation/sparc/adi.txt b/Documentation/sparc/adi.txt new file mode 100644 index 000000000000..e1aed155fb89 --- /dev/null +++ b/Documentation/sparc/adi.txt @@ -0,0 +1,278 @@ +Application Data Integrity (ADI) +================================ + +SPARC M7 processor adds the Application Data Integrity (ADI) feature. +ADI allows a task to set version tags on any subset of its address +space. Once ADI is enabled and version tags are set for ranges of +address space of a task, the processor will compare the tag in pointers +to memory in these ranges to the version set by the application +previously. Access to memory is granted only if the tag in given pointer +matches the tag set by the application. In case of mismatch, processor +raises an exception. + +Following steps must be taken by a task to enable ADI fully: + +1. Set the user mode PSTATE.mcde bit. This acts as master switch for + the task's entire address space to enable/disable ADI for the task. + +2. Set TTE.mcd bit on any TLB entries that correspond to the range of + addresses ADI is being enabled on. MMU checks the version tag only + on the pages that have TTE.mcd bit set. + +3. Set the version tag for virtual addresses using stxa instruction + and one of the MCD specific ASIs. Each stxa instruction sets the + given tag for one ADI block size number of bytes. This step must + be repeated for entire page to set tags for entire page. + +ADI block size for the platform is provided by the hypervisor to kernel +in machine description tables. Hypervisor also provides the number of +top bits in the virtual address that specify the version tag. Once +version tag has been set for a memory location, the tag is stored in the +physical memory and the same tag must be present in the ADI version tag +bits of the virtual address being presented to the MMU. For example on +SPARC M7 processor, MMU uses bits 63-60 for version tags and ADI block +size is same as cacheline size which is 64 bytes. A task that sets ADI +version to, say 10, on a range of memory, must access that memory using +virtual addresses that contain 0xa in bits 63-60. + +ADI is enabled on a set of pages using mprotect() with PROT_ADI flag. +When ADI is enabled on a set of pages by a task for the first time, +kernel sets the PSTATE.mcde bit fot the task. Version tags for memory +addresses are set with an stxa instruction on the addresses using +ASI_MCD_PRIMARY or ASI_MCD_ST_BLKINIT_PRIMARY. ADI block size is +provided by the hypervisor to the kernel. Kernel returns the value of +ADI block size to userspace using auxiliary vector along with other ADI +info. Following auxiliary vectors are provided by the kernel: + + AT_ADI_BLKSZ ADI block size. This is the granularity and + alignment, in bytes, of ADI versioning. + AT_ADI_NBITS Number of ADI version bits in the VA + + +IMPORTANT NOTES: + +- Version tag values of 0x0 and 0xf are reserved. These values match any + tag in virtual address and never generate a mismatch exception. + +- Version tags are set on virtual addresses from userspace even though + tags are stored in physical memory. Tags are set on a physical page + after it has been allocated to a task and a pte has been created for + it. + +- When a task frees a memory page it had set version tags on, the page + goes back to free page pool. When this page is re-allocated to a task, + kernel clears the page using block initialization ASI which clears the + version tags as well for the page. If a page allocated to a task is + freed and allocated back to the same task, old version tags set by the + task on that page will no longer be present. + +- ADI tag mismatches are not detected for non-faulting loads. + +- Kernel does not set any tags for user pages and it is entirely a + task's responsibility to set any version tags. Kernel does ensure the + version tags are preserved if a page is swapped out to the disk and + swapped back in. It also preserves that version tags if a page is + migrated. + +- ADI works for any size pages. A userspace task need not be aware of + page size when using ADI. It can simply select a virtual address + range, enable ADI on the range using mprotect() and set version tags + for the entire range. mprotect() ensures range is aligned to page size + and is a multiple of page size. + +- ADI tags can only be set on writable memory. For example, ADI tags can + not be set on read-only mappings. + + + +ADI related traps +----------------- + +With ADI enabled, following new traps may occur: + +Disrupting memory corruption + + When a store accesses a memory localtion that has TTE.mcd=1, + the task is running with ADI enabled (PSTATE.mcde=1), and the ADI + tag in the address used (bits 63:60) does not match the tag set on + the corresponding cacheline, a memory corruption trap occurs. By + default, it is a disrupting trap and is sent to the hypervisor + first. Hypervisor creates a sun4v error report and sends a + resumable error (TT=0x7e) trap to the kernel. The kernel sends + a SIGSEGV to the task that resulted in this trap with the following + info: + + siginfo.si_signo = SIGSEGV; + siginfo.errno = 0; + siginfo.si_code = SEGV_ADIDERR; + siginfo.si_addr = addr; /* PC where first mismatch occurred */ + siginfo.si_trapno = 0; + + +Precise memory corruption + + When a store accesses a memory location that has TTE.mcd=1, + the task is running with ADI enabled (PSTATE.mcde=1), and the ADI + tag in the address used (bits 63:60) does not match the tag set on + the corresponding cacheline, a memory corruption trap occurs. If + MCD precise exception is enabled (MCDPERR=1), a precise + exception is sent to the kernel with TT=0x1a. The kernel sends + a SIGSEGV to the task that resulted in this trap with the following + info: + + siginfo.si_signo = SIGSEGV; + siginfo.errno = 0; + siginfo.si_code = SEGV_ADIPERR; + siginfo.si_addr = addr; /* address that caused trap */ + siginfo.si_trapno = 0; + + NOTE: ADI tag mismatch on a load always results in precise trap. + + +MCD disabled + + When a task has not enabled ADI and attempts to set ADI version + on a memory address, processor sends an MCD disabled trap. This + trap is handled by hypervisor first and the hypervisor vectors this + trap through to the kernel as Data Access Exception trap with + fault type set to 0xa (invalid ASI). When this occurs, the kernel + sends the task SIGSEGV signal with following info: + + siginfo.si_signo = SIGSEGV; + siginfo.errno = 0; + siginfo.si_code = SEGV_ACCADI; + siginfo.si_addr = addr; /* address that caused trap */ + siginfo.si_trapno = 0; + + +Sample program to use ADI +------------------------- + +Following sample program is meant to illustrate how to use the ADI +functionality. + +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <elf.h> +#include <sys/ipc.h> +#include <sys/shm.h> +#include <sys/mman.h> +#include <asm/asi.h> + +#ifndef AT_ADI_BLKSZ +#define AT_ADI_BLKSZ 48 +#endif +#ifndef AT_ADI_NBITS +#define AT_ADI_NBITS 49 +#endif + +#ifndef PROT_ADI +#define PROT_ADI 0x10 +#endif + +#define BUFFER_SIZE 32*1024*1024UL + +main(int argc, char* argv[], char* envp[]) +{ + unsigned long i, mcde, adi_blksz, adi_nbits; + char *shmaddr, *tmp_addr, *end, *veraddr, *clraddr; + int shmid, version; + Elf64_auxv_t *auxv; + + adi_blksz = 0; + + while(*envp++ != NULL); + for (auxv = (Elf64_auxv_t *)envp; auxv->a_type != AT_NULL; auxv++) { + switch (auxv->a_type) { + case AT_ADI_BLKSZ: + adi_blksz = auxv->a_un.a_val; + break; + case AT_ADI_NBITS: + adi_nbits = auxv->a_un.a_val; + break; + } + } + if (adi_blksz == 0) { + fprintf(stderr, "Oops! ADI is not supported\n"); + exit(1); + } + + printf("ADI capabilities:\n"); + printf("\tBlock size = %ld\n", adi_blksz); + printf("\tNumber of bits = %ld\n", adi_nbits); + + if ((shmid = shmget(2, BUFFER_SIZE, + IPC_CREAT | SHM_R | SHM_W)) < 0) { + perror("shmget failed"); + exit(1); + } + + shmaddr = shmat(shmid, NULL, 0); + if (shmaddr == (char *)-1) { + perror("shm attach failed"); + shmctl(shmid, IPC_RMID, NULL); + exit(1); + } + + if (mprotect(shmaddr, BUFFER_SIZE, PROT_READ|PROT_WRITE|PROT_ADI)) { + perror("mprotect failed"); + goto err_out; + } + + /* Set the ADI version tag on the shm segment + */ + version = 10; + tmp_addr = shmaddr; + end = shmaddr + BUFFER_SIZE; + while (tmp_addr < end) { + asm volatile( + "stxa %1, [%0]0x90\n\t" + : + : "r" (tmp_addr), "r" (version)); + tmp_addr += adi_blksz; + } + asm volatile("membar #Sync\n\t"); + + /* Create a versioned address from the normal address by placing + * version tag in the upper adi_nbits bits + */ + tmp_addr = (void *) ((unsigned long)shmaddr << adi_nbits); + tmp_addr = (void *) ((unsigned long)tmp_addr >> adi_nbits); + veraddr = (void *) (((unsigned long)version << (64-adi_nbits)) + | (unsigned long)tmp_addr); + + printf("Starting the writes:\n"); + for (i = 0; i < BUFFER_SIZE; i++) { + veraddr[i] = (char)(i); + if (!(i % (1024 * 1024))) + printf("."); + } + printf("\n"); + + printf("Verifying data..."); + fflush(stdout); + for (i = 0; i < BUFFER_SIZE; i++) + if (veraddr[i] != (char)i) + printf("\nIndex %lu mismatched\n", i); + printf("Done.\n"); + + /* Disable ADI and clean up + */ + if (mprotect(shmaddr, BUFFER_SIZE, PROT_READ|PROT_WRITE)) { + perror("mprotect failed"); + goto err_out; + } + + if (shmdt((const void *)shmaddr) != 0) + perror("Detach failure"); + shmctl(shmid, IPC_RMID, NULL); + + exit(0); + +err_out: + if (shmdt((const void *)shmaddr) != 0) + perror("Detach failure"); + shmctl(shmid, IPC_RMID, NULL); + exit(1); +} diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h index 07e3f54de9e3..e3f1b5ba5d5c 100644 --- a/arch/powerpc/include/asm/mman.h +++ b/arch/powerpc/include/asm/mman.h @@ -43,7 +43,7 @@ static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags) } #define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags) -static inline bool arch_validate_prot(unsigned long prot) +static inline bool arch_validate_prot(unsigned long prot, unsigned long addr) { if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_SAO)) return false; @@ -51,7 +51,7 @@ static inline bool arch_validate_prot(unsigned long prot) return false; return true; } -#define arch_validate_prot(prot) arch_validate_prot(prot) +#define arch_validate_prot arch_validate_prot #endif /* CONFIG_PPC64 */ #endif /* _ASM_POWERPC_MMAN_H */ diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.c index 1ef3b80b62a6..466216506eb2 100644 --- a/arch/powerpc/kernel/syscalls.c +++ b/arch/powerpc/kernel/syscalls.c @@ -48,7 +48,7 @@ static inline long do_mmap2(unsigned long addr, size_t len, { long ret = -EINVAL; - if (!arch_validate_prot(prot)) + if (!arch_validate_prot(prot, addr)) goto out; if (shift) { diff --git a/arch/sparc/include/asm/adi.h b/arch/sparc/include/asm/adi.h new file mode 100644 index 000000000000..acad0d04e4c6 --- /dev/null +++ b/arch/sparc/include/asm/adi.h @@ -0,0 +1,6 @@ +#ifndef ___ASM_SPARC_ADI_H +#define ___ASM_SPARC_ADI_H +#if defined(__sparc__) && defined(__arch64__) +#include <asm/adi_64.h> +#endif +#endif diff --git a/arch/sparc/include/asm/adi_64.h b/arch/sparc/include/asm/adi_64.h new file mode 100644 index 000000000000..85f7a763af85 --- /dev/null +++ b/arch/sparc/include/asm/adi_64.h @@ -0,0 +1,47 @@ +/* adi_64.h: ADI related data structures + * + * Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. + * Author: Khalid Aziz (khalid.aziz@oracle.com) + * + * This work is licensed under the terms of the GNU GPL, version 2. + */ +#ifndef __ASM_SPARC64_ADI_H +#define __ASM_SPARC64_ADI_H + +#include <linux/types.h> + +#ifndef __ASSEMBLY__ + +struct adi_caps { + __u64 blksz; + __u64 nbits; + __u64 ue_on_adi; +}; + +struct adi_config { + bool enabled; + struct adi_caps caps; +}; + +extern struct adi_config adi_state; + +extern void mdesc_adi_init(void); + +static inline bool adi_capable(void) +{ + return adi_state.enabled; +} + +static inline unsigned long adi_blksize(void) +{ + return adi_state.caps.blksz; +} + +static inline unsigned long adi_nbits(void) +{ + return adi_state.caps.nbits; +} + +#endif /* __ASSEMBLY__ */ + +#endif /* !(__ASM_SPARC64_ADI_H) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index abad97edf736..28db058d471b 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -83,7 +83,11 @@ ATOMIC_OPS(xor) #define atomic64_add_negative(i, v) (atomic64_add_return(i, v) < 0) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) + +static inline int atomic_xchg(atomic_t *v, int new) +{ + return xchg(&v->counter, new); +} static inline int __atomic_add_unless(atomic_t *v, int a, int u) { diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h index 25340df3570c..7e078bc73ef5 100644 --- a/arch/sparc/include/asm/elf_64.h +++ b/arch/sparc/include/asm/elf_64.h @@ -10,6 +10,7 @@ #include <asm/processor.h> #include <asm/extable_64.h> #include <asm/spitfire.h> +#include <asm/adi.h> /* * Sparc section types @@ -215,9 +216,13 @@ extern unsigned int vdso_enabled; #define ARCH_DLINFO \ do { \ + extern struct adi_config adi_state; \ if (vdso_enabled) \ NEW_AUX_ENT(AT_SYSINFO_EHDR, \ (unsigned long)current->mm->context.vdso); \ + NEW_AUX_ENT(AT_ADI_BLKSZ, adi_state.caps.blksz); \ + NEW_AUX_ENT(AT_ADI_NBITS, adi_state.caps.nbits); \ + NEW_AUX_ENT(AT_ADI_UEONADI, adi_state.caps.ue_on_adi); \ } while (0) struct linux_binprm; diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h index ab9c6b027b75..08650d503cc2 100644 --- a/arch/sparc/include/asm/hypervisor.h +++ b/arch/sparc/include/asm/hypervisor.h @@ -570,6 +570,8 @@ struct hv_fault_status { #define HV_FAULT_TYPE_RESV1 13 #define HV_FAULT_TYPE_UNALIGNED 14 #define HV_FAULT_TYPE_INV_PGSZ 15 +#define HV_FAULT_TYPE_MCD 17 +#define HV_FAULT_TYPE_MCD_DIS 18 /* Values 16 --> -2 are reserved. */ #define HV_FAULT_TYPE_MULTIPLE -1 diff --git a/arch/sparc/include/asm/mman.h b/arch/sparc/include/asm/mman.h index 7e9472143f9b..f94532f25db1 100644 --- a/arch/sparc/include/asm/mman.h +++ b/arch/sparc/include/asm/mman.h @@ -7,5 +7,87 @@ #ifndef __ASSEMBLY__ #define arch_mmap_check(addr,len,flags) sparc_mmap_check(addr,len) int sparc_mmap_check(unsigned long addr, unsigned long len); -#endif + +#ifdef CONFIG_SPARC64 +#include <asm/adi_64.h> + +static inline void ipi_set_tstate_mcde(void *arg) +{ + struct mm_struct *mm = arg; + + /* Set TSTATE_MCDE for the task using address map that ADI has been + * enabled on if the task is running. If not, it will be set + * automatically at the next context switch + */ + if (current->mm == mm) { + struct pt_regs *regs; + + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + } +} + +#define arch_calc_vm_prot_bits(prot, pkey) sparc_calc_vm_prot_bits(prot) +static inline unsigned long sparc_calc_vm_prot_bits(unsigned long prot) +{ + if (adi_capable() && (prot & PROT_ADI)) { + struct pt_regs *regs; + + if (!current->mm->context.adi) { + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + current->mm->context.adi = true; + on_each_cpu_mask(mm_cpumask(current->mm), + ipi_set_tstate_mcde, current->mm, 0); + } + return VM_SPARC_ADI; + } else { + return 0; + } +} + +#define arch_vm_get_page_prot(vm_flags) sparc_vm_get_page_prot(vm_flags) +static inline pgprot_t sparc_vm_get_page_prot(unsigned long vm_flags) +{ + return (vm_flags & VM_SPARC_ADI) ? __pgprot(_PAGE_MCD_4V) : __pgprot(0); +} + +#define arch_validate_prot(prot, addr) sparc_validate_prot(prot, addr) +static inline int sparc_validate_prot(unsigned long prot, unsigned long addr) +{ + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_ADI)) + return 0; + if (prot & PROT_ADI) { + if (!adi_capable()) + return 0; + + if (addr) { + struct vm_area_struct *vma; + + vma = find_vma(current->mm, addr); + if (vma) { + /* ADI can not be enabled on PFN + * mapped pages + */ + if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + return 0; + + /* Mergeable pages can become unmergeable + * if ADI is enabled on them even if they + * have identical data on them. This can be + * because ADI enabled pages with identical + * data may still not have identical ADI + * tags on them. Disallow ADI on mergeable + * pages. + */ + if (vma->vm_flags & VM_MERGEABLE) + return 0; + } + } + } + return 1; +} +#endif /* CONFIG_SPARC64 */ + +#endif /* __ASSEMBLY__ */ #endif /* __SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h index ad4fb93508ba..7e2704c770e9 100644 --- a/arch/sparc/include/asm/mmu_64.h +++ b/arch/sparc/include/asm/mmu_64.h @@ -90,6 +90,20 @@ struct tsb_config { #define MM_NUM_TSBS 1 #endif +/* ADI tags are stored when a page is swapped out and the storage for + * tags is allocated dynamically. There is a tag storage descriptor + * associated with each set of tag storage pages. Tag storage descriptors + * are allocated dynamically. Since kernel will allocate a full page for + * each tag storage descriptor, we can store up to + * PAGE_SIZE/sizeof(tag storage descriptor) descriptors on that page. + */ +typedef struct { + unsigned long start; /* Start address for this tag storage */ + unsigned long end; /* Last address for tag storage */ + unsigned char *tags; /* Where the tags are */ + unsigned long tag_users; /* number of references to descriptor */ +} tag_storage_desc_t; + typedef struct { spinlock_t lock; unsigned long sparc64_ctx_val; @@ -98,6 +112,9 @@ typedef struct { struct tsb_config tsb_block[MM_NUM_TSBS]; struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; void *vdso; + bool adi; + tag_storage_desc_t *tag_store; + spinlock_t tag_lock; } mm_context_t; #endif /* !__ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index b361702ef52a..312fcee8df2b 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -9,8 +9,10 @@ #include <linux/spinlock.h> #include <linux/mm_types.h> #include <linux/smp.h> +#include <linux/sched.h> #include <asm/spitfire.h> +#include <asm/adi_64.h> #include <asm-generic/mm_hooks.h> #include <asm/percpu.h> @@ -136,6 +138,55 @@ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, str #define deactivate_mm(tsk,mm) do { } while (0) #define activate_mm(active_mm, mm) switch_mm(active_mm, mm, NULL) + +#define __HAVE_ARCH_START_CONTEXT_SWITCH +static inline void arch_start_context_switch(struct task_struct *prev) +{ + /* Save the current state of MCDPER register for the process + * we are switching from + */ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + __asm__ __volatile__( + ".word 0x83438000\n\t" /* rd %mcdper, %g1 */ + "mov %%g1, %0\n\t" + : "=r" (tmp_mcdper) + : + : "g1"); + if (tmp_mcdper) + set_tsk_thread_flag(prev, TIF_MCDPER); + else + clear_tsk_thread_flag(prev, TIF_MCDPER); + } +} + +#define finish_arch_post_lock_switch finish_arch_post_lock_switch +static inline void finish_arch_post_lock_switch(void) +{ + /* Restore the state of MCDPER register for the new process + * just switched to. + */ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + tmp_mcdper = test_thread_flag(TIF_MCDPER); + __asm__ __volatile__( + "mov %0, %%g1\n\t" + ".word 0x9d800001\n\t" /* wr %g0, %g1, %mcdper" */ + ".word 0xaf902001\n\t" /* wrpr %g0, 1, %pmcdper */ + : + : "ir" (tmp_mcdper) + : "g1"); + if (current && current->mm && current->mm->context.adi) { + struct pt_regs *regs; + + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + } + } +} + #endif /* !(__ASSEMBLY__) */ #endif /* !(__SPARC64_MMU_CONTEXT_H) */ diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index c28379b1b0fc..e80f2d5bf62f 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h @@ -48,6 +48,12 @@ struct page; void clear_user_page(void *addr, unsigned long vaddr, struct page *page); #define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE) void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); +#define __HAVE_ARCH_COPY_USER_HIGHPAGE +struct vm_area_struct; +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +#define __HAVE_ARCH_COPY_HIGHPAGE +void copy_highpage(struct page *to, struct page *from); /* Unlike sparc32, sparc64's parameter passing API is more * sane in that structures which as small enough are passed |