path: root/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2019 Mellanox Technologies.

#include <net/inet6_hashtables.h>
#include "en_accel/en_accel.h"
#include "en_accel/tls.h"
#include "en_accel/ktls_txrx.h"
#include "en_accel/ktls_utils.h"
#include "en_accel/fs_tcp.h"

struct accel_rule {
	struct work_struct work;
	struct mlx5e_priv *priv;
	struct mlx5_flow_handle *rule;
};

#define PROGRESS_PARAMS_WRITE_UNIT	64
#define PROGRESS_PARAMS_PADDED_SIZE	\
		(ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \
		       PROGRESS_PARAMS_WRITE_UNIT))

struct mlx5e_ktls_rx_resync_buf {
	union {
		struct mlx5_wqe_tls_progress_params_seg progress;
		u8 pad[PROGRESS_PARAMS_PADDED_SIZE];
	} ____cacheline_aligned_in_smp;
	dma_addr_t dma_addr;
	struct mlx5e_ktls_offload_context_rx *priv_rx;
};

enum {
	MLX5E_PRIV_RX_FLAG_DELETING,
	MLX5E_NUM_PRIV_RX_FLAGS,
};

struct mlx5e_ktls_rx_resync_ctx {
	struct tls_offload_resync_async core;
	struct work_struct work;
	struct mlx5e_priv *priv;
	refcount_t refcnt;
	__be64 sw_rcd_sn_be;
	u32 seq;
};

struct mlx5e_ktls_offload_context_rx {
	struct tls12_crypto_info_aes_gcm_128 crypto_info;
	struct accel_rule rule;
	struct sock *sk;
	struct mlx5e_rq_stats *stats;
	struct completion add_ctx;
	u32 tirn;
	u32 key_id;
	u32 rxq;
	DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);

	/* resync */
	struct mlx5e_ktls_rx_resync_ctx resync;
};

static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, u32 *tirn, u32 rqtn)
{
	int err, inlen;
	void *tirc;
	u32 *in;

	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);

	MLX5_SET(tirc, tirc, transport_domain, mdev->mlx5e_res.td.tdn);
	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
	MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
	MLX5_SET(tirc, tirc, indirect_table, rqtn);
	MLX5_SET(tirc, tirc, tls_en, 1);
	MLX5_SET(tirc, tirc, self_lb_block,
		 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST |
		 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST);

	err = mlx5_core_create_tir(mdev, in, tirn);

	kvfree(in);
	return err;
}

static void accel_rule_handle_work(struct work_struct *work)
{
	struct mlx5e_ktls_offload_context_rx *priv_rx;
	struct accel_rule *accel_rule;
	struct mlx5_flow_handle *rule;

	accel_rule = container_of(work, struct accel_rule, work);
	priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule);
	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
		goto out;

	rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk,
				     priv_rx->tirn, MLX5_FS_DEFAULT_FLOW_TAG);
	if (!IS_ERR_OR_NULL(rule))
		accel_rule->rule = rule;
out:
	complete(&priv_rx->add_ctx);
}

static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv,
			    struct sock *sk)
{
	INIT_WORK(&rule->work, accel_rule_handle_work);
	rule->priv = priv;
}

static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi,
			  struct mlx5e_icosq_wqe_info *wi)
{
	sq->db.wqe_info[pi] = *wi;
}

static struct mlx5_wqe_ctrl_seg *
post_static_params(struct mlx5e_icosq *sq,
		   struct mlx5e_ktls_offload_context_rx *priv_rx)
{
	struct mlx5e_set_tls_static_params_wqe *wqe;
	struct mlx5e_icosq_wqe_info wi;
	u16 pi, num_wqebbs, room;

	num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
	room = mlx5e_stop_room_for_wqe(num_wqebbs);
	if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room)))
		return ERR_PTR(-ENOSPC);

	pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
	wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
	mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info,
				       priv_rx->tirn, priv_rx->key_id,
				       priv_rx->resync.seq, false,
				       TLS_OFFLOAD_CTX_DIR_RX);
	wi = (struct mlx5e_icosq_wqe_info) {
		.wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS,
		.num_wqebbs = num_wqebbs,
		.tls_set_params.priv_rx = priv_rx,
	};
	icosq_fill_wi(sq, pi, &wi);
	sq->pc += num_wqebbs;

	return &wqe->ctrl;
}

static struct mlx5_wqe_ctrl_seg *
post_progress_params(struct mlx5e_icosq *sq,
		     struct mlx5e_ktls_offload_context_rx *priv_rx,
		     u32 next_record_tcp_sn)
{
	struct mlx5e_set_tls_progress_params_wqe *wqe;
	struct mlx5e_icosq_wqe_info wi;
	u16 pi, num_wqebbs, room;

	num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
	room = mlx5e_stop_room_for_wqe(num_wqebbs);
	if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room)))
		return ERR_PTR(-ENOSPC);

	pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
	wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
	mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_rx->tirn, false,
					 next_record_tcp_sn,
					 TLS_OFFLOAD_CTX_DIR_RX);
	wi = (struct mlx5e_icosq_wqe_info) {
		.wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS,
		.num_wqebbs = num_wqebbs,
		.tls_set_params.priv_rx = priv_rx,
	};

	icosq_fill_wi(sq, pi, &wi);
	sq->pc += num_wqebbs;

	return &wqe->ctrl;
}

static int post_rx_param_wqes(struct mlx5e_channel *c,
			      struct mlx5e_ktls_offload_context_rx *priv_rx,
			      u32 next_record_tcp_sn)
{
	struct mlx5_wqe_ctrl_seg *cseg;
	struct mlx5e_icosq *sq;