path: root/arch/mips/kernel/smp.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *
 * Copyright (C) 2000, 2001 Kanoj Sarcar
 * Copyright (C) 2000, 2001 Ralf Baechle
 * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation
 */
#include <linux/cache.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/threads.h>
#include <linux/export.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/sched/mm.h>
#include <linux/cpumask.h>
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/ftrace.h>
#include <linux/irqdomain.h>
#include <linux/of.h>
#include <linux/of_irq.h>

#include <linux/atomic.h>
#include <asm/cpu.h>
#include <asm/ginvt.h>
#include <asm/processor.h>
#include <asm/idle.h>
#include <asm/r4k-timer.h>
#include <asm/mips-cps.h>
#include <asm/mmu_context.h>
#include <asm/time.h>
#include <asm/setup.h>
#include <asm/maar.h>

int __cpu_number_map[CONFIG_MIPS_NR_CPU_NR_MAP];   /* Map physical to logical */
EXPORT_SYMBOL(__cpu_number_map);

int __cpu_logical_map[NR_CPUS];		/* Map logical to physical */
EXPORT_SYMBOL(__cpu_logical_map);

/* Number of TCs (or siblings in Intel speak) per CPU core */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* representing the TCs (or siblings in Intel speak) of each logical CPU */
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(cpu_sibling_map);

/* representing the core map of multi-core chips of each logical CPU */
cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(cpu_core_map);

static DECLARE_COMPLETION(cpu_starting);
static DECLARE_COMPLETION(cpu_running);

/*
 * A logcal cpu mask containing only one VPE per core to
 * reduce the number of IPIs on large MT systems.
 */
cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(cpu_foreign_map);

/* representing cpus for which sibling maps can be computed */
static cpumask_t cpu_sibling_setup_map;

/* representing cpus for which core maps can be computed */
static cpumask_t cpu_core_setup_map;

cpumask_t cpu_coherent_mask;

#ifdef CONFIG_GENERIC_IRQ_IPI
static struct irq_desc *call_desc;
static struct irq_desc *sched_desc;
#endif

static inline void set_cpu_sibling_map(int cpu)
{
	int i;

	cpumask_set_cpu(cpu, &cpu_sibling_setup_map);

	if (smp_num_siblings > 1) {
		for_each_cpu(i, &cpu_sibling_setup_map) {
			if (cpus_are_siblings(cpu, i)) {
				cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
				cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
			}
		}
	} else
		cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]);
}

static inline void set_cpu_core_map(int cpu)
{
	int i;

	cpumask_set_cpu(cpu, &cpu_core_setup_map);

	for_each_cpu(i, &cpu_core_setup_map) {
		if (cpu_data[cpu].package == cpu_data[i].package) {
			cpumask_set_cpu(i, &cpu_core_map[cpu]);
			cpumask_set_cpu(cpu, &cpu_core_map[i]);
		}
	}
}

/*
 * Calculate a new cpu_foreign_map mask whenever a
 * new cpu appears or disappears.
 */
void calculate_cpu_foreign_map(void)
{
	int i, k, core_present;
	cpumask_t temp_foreign_map;

	/* Re-calculate the mask */
	cpumask_clear(&temp_foreign_map);
	for_each_online_cpu(i) {
		core_present = 0;
		for_each_cpu(k, &temp_foreign_map)
			if (cpus_are_siblings(i, k))
				core_present = 1;
		if (!core_present)
			cpumask_set_cpu(i, &temp_foreign_map);
	}

	for_each_online_cpu(i)
		cpumask_andnot(&cpu_foreign_map[i],
			       &temp_foreign_map, &cpu_sibling_map[i]);
}

const struct plat_smp_ops *mp_ops;
EXPORT_SYMBOL(mp_ops);

void register_smp_ops(const struct plat_smp_ops *ops)
{
	if (mp_ops)
		printk(KERN_WARNING "Overriding previously set SMP ops\n");

	mp_ops = ops;
}

#ifdef CONFIG_GENERIC_IRQ_IPI
void mips_smp_send_ipi_single(int cpu, unsigned int action)
{
	mips_smp_send_ipi_mask(cpumask_of(cpu), action);
}

void mips_smp_send_ipi_mask(const struct cpumask *mask, unsigned int action)
{
	unsigned long flags;
	unsigned int core;
	int cpu;

	local_irq_save(flags);

	switch (action) {
	case SMP_CALL_FUNCTION:
		__ipi_send_mask(call_desc, mask);
		break;

	case SMP_RESCHEDULE_YOURSELF:
		__ipi_send_mask(sched_desc, mask);
		break;

	default:
		BUG();
	}

	if (mips_cpc_present()) {
		for_each_cpu(cpu, mask) {
			if (cpus_are_siblings(cpu, smp_processor_id()))
				continue;

			core = cpu_core(&cpu_data[cpu]);

			while (!cpumask_test_cpu(cpu, &cpu_coherent_mask)) {
				mips_cm_lock_other_cpu(cpu, CM_GCR_Cx_OTHER_BLOCK_LOCAL);
				mips_cpc_lock_other(core);
				write_cpc_co_cmd(CPC_Cx_CMD_PWRUP);
				mips_cpc_unlock_other();
				mips_cm_unlock_other();
			}
		}
	}

	local_irq_restore(flags);
}


static irqreturn_t ipi_resched_interrupt(int irq, void *dev_id)
{
	scheduler_ipi();

	return IRQ_HANDLED;
}

static irqreturn_t ipi_call_interrupt(int irq, void *dev_id)
{
	generic_smp_call_function_interrupt();

	return IRQ_HANDLED;
}

static void smp_ipi_init_one(unsigned int virq, const char *name,
			     irq_handler_t handler)
{
	int ret;

	irq_set_handler(virq, handle_percpu_irq);
	ret = request_irq(virq, handler, IRQF_PERCPU, name, NULL);
	BUG_ON(ret);
}

static unsigned int call_virq, sched_virq;

int mips_smp_ipi_allocate(const struct cpumask *mask)
{
	int virq;
	struct irq_domain *ipidomain;
	struct device_node *node;

	node = of_irq_find_parent(of_root);
	ipidomain = irq_find_matching_host(node, DOMAIN_BUS_IPI);

	/*
	 * Some platforms have half DT setup. So if we found irq node but
	 * didn't find an ipidomain, try to search for one that is not in the
	 * DT.
	 */
	if (node && !ipidomain)
		ipidomain = irq_find_matching_host(NULL, DOMAIN_BUS_IPI);

	/*
	 * There are systems which use IPI IRQ domains, but only have one
	 * registered when some runtime condition is met. For example a Malta
	 * kernel may include support for GIC & CPU interrupt controller IPI
	 * IRQ domains, but if run on a system with no GIC & no MT ASE then
	 * neither will be supported or registered.
	 *
	 * We only have a problem if we're actually using multiple CPUs so fail
	 * loudly if that is the case. Otherwise simply return, skipping IPI
	 * setup, if we're running with only a single CPU.
	 */
	if (!ipidomain) {
		BUG_ON(num_present_cpus() > 1);
		return 0;
	}

	virq = irq_reserve_ipi(ipidomain, mask);
	BUG_ON(!virq);
	if (!call_virq)
		call_virq = virq;

	virq = irq_reserve_ipi(ipidomain, mask);
	BUG_ON(!virq);
	if (!sched_virq)
		sched_virq = virq;

	if (irq_domain_is_ipi_per_cpu(ipidomain)) {
		int