// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2020 Mellanox Technologies.
#include <linux/mlx5/driver.h>
#include <linux/mlx5/mlx5_ifc.h>
#include <linux/mlx5/fs.h>
#include "esw/chains.h"
#include "en/mapping.h"
#include "mlx5_core.h"
#include "fs_core.h"
#include "eswitch.h"
#include "en.h"
#include "en_tc.h"
#define esw_chains_priv(esw) ((esw)->fdb_table.offloads.esw_chains_priv)
#define esw_chains_lock(esw) (esw_chains_priv(esw)->lock)
#define esw_chains_ht(esw) (esw_chains_priv(esw)->chains_ht)
#define esw_chains_mapping(esw) (esw_chains_priv(esw)->chains_mapping)
#define esw_prios_ht(esw) (esw_chains_priv(esw)->prios_ht)
#define fdb_pool_left(esw) (esw_chains_priv(esw)->fdb_left)
#define tc_slow_fdb(esw) ((esw)->fdb_table.offloads.slow_fdb)
#define tc_end_fdb(esw) (esw_chains_priv(esw)->tc_end_fdb)
#define fdb_ignore_flow_level_supported(esw) \
(MLX5_CAP_ESW_FLOWTABLE_FDB((esw)->dev, ignore_flow_level))
#define fdb_modify_header_fwd_to_table_supported(esw) \
(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
/* Firmware currently has 4 pool of 4 sizes that it supports (ESW_POOLS),
* and a virtual memory region of 16M (ESW_SIZE), this region is duplicated
* for each flow table pool. We can allocate up to 16M of each pool,
* and we keep track of how much we used via get_next_avail_sz_from_pool.
* Firmware doesn't report any of this for now.
* ESW_POOL is expected to be sorted from large to small and match firmware
* pools.
*/
#define ESW_SIZE (16 * 1024 * 1024)
static const unsigned int ESW_POOLS[] = { 4 * 1024 * 1024,
1 * 1024 * 1024,
64 * 1024,
128 };
#define ESW_FT_TBL_SZ (64 * 1024)
struct mlx5_esw_chains_priv {
struct rhashtable chains_ht;
struct rhashtable prios_ht;
/* Protects above chains_ht and prios_ht */
struct mutex lock;
struct mlx5_flow_table *tc_end_fdb;
struct mapping_ctx *chains_mapping;
int fdb_left[ARRAY_SIZE(ESW_POOLS)];
};
struct fdb_chain {
struct rhash_head node;
u32 chain;
int ref;
int id;
struct mlx5_eswitch *esw;
struct list_head prios_list;
struct mlx5_flow_handle *restore_rule;
struct mlx5_modify_hdr *miss_modify_hdr;
};
struct fdb_prio_key {
u32 chain;
u32 prio;
u32 level;
};
struct fdb_prio {
struct rhash_head node;
struct list_head list;
struct fdb_prio_key key;
int ref;
struct fdb_chain *fdb_chain;
struct mlx5_flow_table *fdb;
struct mlx5_flow_table *next_fdb;
struct mlx5_flow_group *miss_group;
struct mlx5_flow_handle *miss_rule;
};
static const struct rhashtable_params chain_params = {
.head_offset = offsetof(struct fdb_chain, node),
.key_offset = offsetof(struct fdb_chain, chain),
.key_len = sizeof_field(struct fdb_chain, chain),
.automatic_shrinking = true,
};
static const struct rhashtable_params prio_params = {
.head_offset = offsetof(struct fdb_prio, node),
.key_offset = offsetof(struct fdb_prio, key),
.key_len = sizeof_field(struct fdb_prio, key),
.automatic_shrinking = true,
};
bool mlx5_esw_chains_prios_supported(struct mlx5_eswitch *esw)
{
return esw->fdb_table.flags & ESW_FDB_CHAINS_AND_PRIOS_SUPPORTED;
}
bool mlx5_esw_chains_backwards_supported(struct mlx5_eswitch *esw)
{
return mlx5_esw_chains_prios_supported(esw) &&
fdb_ignore_flow_level_supported(esw);
}
u32 mlx5_esw_chains_get_chain_range(struct mlx5_eswitch *esw)
{
if (!mlx5_esw_chains_prios_supported(esw))
return 1;
if (fdb_ignore_flow_level_supported(esw))
return UINT_MAX - 1;
return FDB_TC_MAX_CHAIN;
}
u32 mlx5_esw_chains_get_ft_chain(struct mlx5_eswitch *esw)
{
return mlx5_esw_chains_get_chain_range(esw) + 1;
}
u32 mlx5_esw_chains_get_prio_range(struct mlx5_eswitch *esw)
{
if (!mlx5_esw_chains_prios_supported(esw))
return 1;
if (fdb_ignore_flow_level_supported(esw))
return UINT_MAX;
return FDB_TC_MAX_PRIO;
}
static unsigned int mlx5_esw_chains_get_level_range(struct mlx5_eswitch *esw)
{
if (fdb_ignore_flow_level_supported(esw))
return UINT_MAX;
return FDB_TC_LEVELS_PER_PRIO;
}
#define POOL_NEXT_SIZE 0
static int
mlx5_esw_chains_get_avail_sz_from_pool(struct mlx5_eswitch *esw,
int desired_size)
{
int i, found_i = -1;
for (i = ARRAY_SIZE(ESW_POOLS) - 1; i >= 0; i--) {
if (fdb_pool_left(esw)[i] && ESW_POOLS[i] > desired_size) {
found_i = i;
if (desired_size != POOL_NEXT_SIZE)
break;
}
}
if (found_i != -1) {
--fdb_pool_left(esw)[found_i];
return ESW_POOLS[found_i];
}
return 0;
}
static void
mlx5_esw_chains_put_sz_to_pool(struct mlx5_eswitch *esw, int sz)
{
int i;
for (i = ARRAY_SIZE(ESW_POOLS) - 1; i