path: root/arch/s390/mm/kasan_init.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kasan.h>
#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/kasan.h>
#include <asm/mem_detect.h>
#include <asm/processor.h>
#include <asm/sclp.h>
#include <asm/facility.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/uv.h>

unsigned long kasan_vmax;
static unsigned long segment_pos __initdata;
static unsigned long segment_low __initdata;
static unsigned long pgalloc_pos __initdata;
static unsigned long pgalloc_low __initdata;
static unsigned long pgalloc_freeable __initdata;
static bool has_edat __initdata;
static bool has_nx __initdata;

#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))

static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);

static void __init kasan_early_panic(const char *reason)
{
	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	sclp_early_printk(reason);
	disabled_wait();
}

static void * __init kasan_early_alloc_segment(void)
{
	segment_pos -= _SEGMENT_SIZE;

	if (segment_pos < segment_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)segment_pos;
}

static void * __init kasan_early_alloc_pages(unsigned int order)
{
	pgalloc_pos -= (PAGE_SIZE << order);

	if (pgalloc_pos < pgalloc_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)pgalloc_pos;
}

static void * __init kasan_early_crst_alloc(unsigned long val)
{
	unsigned long *table;

	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	if (table)
		crst_table_init(table, val);
	return table;
}

static pte_t * __init kasan_early_pte_alloc(void)
{
	static void *pte_leftover;
	pte_t *pte;

	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);

	if (!pte_leftover) {
		pte_leftover = kasan_early_alloc_pages(0);
		pte = pte_leftover + _PAGE_TABLE_SIZE;
	} else {
		pte = pte_leftover;
		pte_leftover = NULL;
	}
	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	return pte;
}

enum populate_mode {
	POPULATE_ONE2ONE,
	POPULATE_MAP,
	POPULATE_ZERO_SHADOW,
	POPULATE_SHALLOW
};
static void __init kasan_early_pgtable_populate(unsigned long address,
						unsigned long end,
						enum populate_mode mode)
{
	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
	pgd_t *pg_dir;
	p4d_t *p4_dir;
	pud_t *pu_dir;
	pmd_t *pm_dir;
	pte_t *pt_dir;

	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
	if (!has_nx)
		pgt_prot_zero &= ~_PAGE_NOEXEC;
	pgt_prot = pgprot_val(PAGE_KERNEL);
	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	if (!has_nx || mode == POPULATE_ONE2ONE) {
		pgt_prot &= ~_PAGE_NOEXEC;
		sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	}

	while (address < end) {
		pg_dir = pgd_offset_k(address);
		if (pgd_none(*pg_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PGDIR_SIZE) &&
			    end - address >= PGDIR_SIZE) {
				pgd_populate(&init_mm, pg_dir,
						kasan_early_shadow_p4d);
				address = (address + PGDIR_SIZE) & PGDIR_MASK;
				continue;
			}
			p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
			pgd_populate(&init_mm, pg_dir, p4_dir);
		}

		if (mode == POPULATE_SHALLOW) {
			address = (address + P4D_SIZE) & P4D_MASK;
			continue;
		}

		p4_dir = p4d_offset(pg_dir, address);
		if (p4d_none(*p4_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, P4D_SIZE) &&
			    end - address >= P4D_SIZE) {
				p4d_populate(&init_mm, p4_dir,
						kasan_early_shadow_pud);
				address = (address + P4D_SIZE) & P4D_MASK;
				continue;
			}
			pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
			p4d_populate(&init_mm, p4_dir, pu_dir);
		}

		pu_dir = pud_offset(p4_dir, address);
		if (pud_none(*pu_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PUD_SIZE) &&
			    end - address >= PUD_SIZE) {
				pud_populate(&init_mm, pu_dir,
						kasan_early_shadow_pmd);
				address = (address + PUD_SIZE) & PUD_MASK;
				continue;
			}
			pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
			pud_populate(&init_mm, pu_dir, pm_dir);
		}

		pm_dir = pmd_offset(pu_dir, address);
		if (pmd_none(*pm_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PMD_SIZE) &&
			    end - address >= PMD_SIZE) {
				pmd_populate(&init_mm, pm_dir,
						kasan_early_shadow_pte);
				address = (address + PMD_SIZE) & PMD_MASK;
				continue;
			}
			/* the first megabyte of 1:1 is mapped with 4k pages */
			if (has_edat && address && end - address >= PMD_SIZE &&
			    mode != POPULATE_ZERO_SHADOW) {
				void *page;

				if (mode == POPULATE_ONE2ONE) {
					page = (void *)address;
				} else {
					page = kasan_early_alloc_segment();
					memset(page, 0, _SEGMENT_SIZE);
				}
				pmd_val(*pm_dir) = __pa(page) | sgt_prot;
				address = (address + PMD_SIZE) & PMD_MASK;
				continue;
			}

			pt_dir = kasan_early_pte_alloc();
			pmd_populate(&init_mm, pm_dir, pt_dir);
		} else if (pmd_large(*pm_dir)) {
			address = (address + PMD_SIZE) & PMD_MASK;
			continue;
		}

		pt_dir = pte_offset_kernel(pm_dir, address);
		if (pte_none(*pt_dir)) {
			void *page;

			switch (mode) {
			case POPULATE_ONE2ONE:
				page = (void *)address;
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_MAP:
				page = kasan_early_alloc_pages(0);
				memset(page, 0, PAGE_SIZE);
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_ZERO_SHADOW:
				page = kasan_early_shadow_page;
				pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
				break;
			case POPULATE_SHALLOW:
				/* should never happen */
				break;
			}
		}
		address += PAGE_SIZE;
	}
}

static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
{
	unsigned long asce_bits;

	asce_bits = asce_type | _ASCE_TABLE_LENGTH;
	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
	S390_lowcore.user_asce = S390_lowcore.kernel_asce;

	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
}

static void __init kasan_enable_dat(void)
{
	psw_t