/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2017-2018 Intel Corporation
*/
#include <linux/irq.h>
#include <linux/pm_runtime.h>
#include "gt/intel_engine.h"
#include "gt/intel_engine_pm.h"
#include "gt/intel_engine_user.h"
#include "gt/intel_gt_pm.h"
#include "gt/intel_rc6.h"
#include "gt/intel_rps.h"
#include "i915_drv.h"
#include "i915_pmu.h"
#include "intel_pm.h"
/* Frequency for the sampling timer for events which need it. */
#define FREQUENCY 200
#define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
#define ENGINE_SAMPLE_MASK \
(BIT(I915_SAMPLE_BUSY) | \
BIT(I915_SAMPLE_WAIT) | \
BIT(I915_SAMPLE_SEMA))
#define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
static cpumask_t i915_pmu_cpumask;
static u8 engine_config_sample(u64 config)
{
return config & I915_PMU_SAMPLE_MASK;
}
static u8 engine_event_sample(struct perf_event *event)
{
return engine_config_sample(event->attr.config);
}
static u8 engine_event_class(struct perf_event *event)
{
return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
}
static u8 engine_event_instance(struct perf_event *event)
{
return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
}
static bool is_engine_config(u64 config)
{
return config < __I915_PMU_OTHER(0);
}
static unsigned int config_enabled_bit(u64 config)
{
if (is_engine_config(config))
return engine_config_sample(config);
else
return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
}
static u64 config_enabled_mask(u64 config)
{
return BIT_ULL(config_enabled_bit(config));
}
static bool is_engine_event(struct perf_event *event)
{
return is_engine_config(event->attr.config);
}
static unsigned int event_enabled_bit(struct perf_event *event)
{
return config_enabled_bit(event->attr.config);
}
static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active)
{
struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);
u64 enable;
/*
* Only some counters need the sampling timer.
*
* We start with a bitmask of all currently enabled events.
*/
enable = pmu->enable;
/*
* Mask out all the ones which do not need the timer, or in
* other words keep all the ones that could need the timer.
*/
enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
ENGINE_SAMPLE_MASK;
/*
* When the GPU is idle per-engine counters do not need to be
* running so clear those bits out.
*/
if (!gpu_active)
enable &= ~ENGINE_SAMPLE_MASK;
/*
* Also there is software busyness tracking available we do not
* need the timer for I915_SAMPLE_BUSY counter.
*/
else if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS)
enable &= ~BIT(I915_SAMPLE_BUSY);
/*
* If some bits remain it means we need the sampling timer running.
*/
return enable;
}
static u64 __get_rc6(struct intel_gt *gt)
{
struct drm_i915_private *i915 = gt->i915;
u64 val;
val = intel_rc6_residency_ns(>->rc6,
IS_VALLEYVIEW(i915) ?
VLV_GT_RENDER_RC6 :
GEN6_GT_GFX_RC6);
if (HAS_RC6p(i915))
val += intel_rc6_residency_ns(>->rc6, GEN6_GT_GFX_RC6p);
if (HAS_RC6pp