// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Mellanox Technologies. */
#include <linux/smp.h>
#include "dr_types.h"
#define QUEUE_SIZE 128
#define SIGNAL_PER_DIV_QUEUE 16
#define TH_NUMS_TO_DRAIN 2
enum { CQ_OK = 0, CQ_EMPTY = -1, CQ_POLL_ERR = -2 };
struct dr_data_seg {
u64 addr;
u32 length;
u32 lkey;
unsigned int send_flags;
};
struct postsend_info {
struct dr_data_seg write;
struct dr_data_seg read;
u64 remote_addr;
u32 rkey;
};
struct dr_qp_rtr_attr {
struct mlx5dr_cmd_gid_attr dgid_attr;
enum ib_mtu mtu;
u32 qp_num;
u16 port_num;
u8 min_rnr_timer;
u8 sgid_index;
u16 udp_src_port;
};
struct dr_qp_rts_attr {
u8 timeout;
u8 retry_cnt;
u8 rnr_retry;
};
struct dr_qp_init_attr {
u32 cqn;
u32 pdn;
u32 max_send_wr;
struct mlx5_uars_page *uar;
};
static int dr_parse_cqe(struct mlx5dr_cq *dr_cq, struct mlx5_cqe64 *cqe64)
{
unsigned int idx;
u8 opcode;
opcode = get_cqe_opcode(cqe64);
if (opcode == MLX5_CQE_REQ_ERR) {
idx = be16_to_cpu(cqe64->wqe_counter) &
(dr_cq->qp->sq.wqe_cnt - 1);
dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1;
} else if (opcode == MLX5_CQE_RESP_ERR) {
++dr_cq->qp->sq.cc;
} else {
idx = be16_to_cpu(cqe64->wqe_counter) &
(dr_cq->qp->sq.wqe_cnt - 1);
dr_cq->qp->sq.cc = dr_cq->qp->sq.wqe_head[idx] + 1;
return CQ_OK;
}
return CQ_POLL_ERR;
}
static int dr_cq_poll_one(struct mlx5dr_cq *dr_cq)
{
struct mlx5_cqe64 *cqe64;
int err;
cqe64 = mlx5_cqwq_get_cqe(&dr_cq->wq);
if (!cqe64)
return CQ_EMPTY;
mlx5_cqwq_pop(&dr_cq->wq);
err = dr_parse_cqe(dr_cq, cqe64);
mlx5_cqwq_update_db_record(&dr_cq->wq);
return err;
}
static int dr_poll_cq(struct mlx5dr_cq *dr_cq, int ne)
{
int npolled;
int err = 0;
for (npolled = 0; npolled < ne; ++npolled) {
err = dr_cq_poll_one(dr_cq);
if (err != CQ_OK)
break;
}
return err == CQ_POLL_ERR ? err : npolled;
}
static struct mlx5dr_qp *dr_create_rc_qp(struct mlx5_core_dev *mdev,
struct dr_qp_init_attr *attr)
{
u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {};
struct mlx5_wq_param wqp;
struct mlx5dr_qp *dr_qp;
int inlen;
void *qpc;
void *in;
int err;
dr_qp = kzalloc(sizeof(*dr_qp), GFP_KERNEL);
if (!dr_qp)
return NULL;
wqp.buf_numa_node = mdev->priv.numa_node;
wqp.db_numa_node = mdev->priv.numa_node;
dr_qp->rq.pc = 0;
dr_qp->rq.cc = 0;
dr_qp->rq.wqe_cnt = 4;
dr_qp->sq.pc = 0;
dr_qp->sq.cc = 0;
dr_qp->sq.wqe_cnt = roundup_pow_of_two(attr->max_send_wr);
MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(dr_qp->rq.wqe_cnt));
MLX5_SET(qpc,