diff options
Diffstat (limited to 'drivers/staging')
-rw-r--r-- | drivers/staging/media/ipu3/ipu3-css.c | 1740 |
1 files changed, 1740 insertions, 0 deletions
diff --git a/drivers/staging/media/ipu3/ipu3-css.c b/drivers/staging/media/ipu3/ipu3-css.c index 164830fc91ad..3811ad752e8d 100644 --- a/drivers/staging/media/ipu3/ipu3-css.c +++ b/drivers/staging/media/ipu3/ipu3-css.c @@ -16,6 +16,173 @@ IMGU_IRQCTRL_IRQ_SW_PIN(0) | \ IMGU_IRQCTRL_IRQ_SW_PIN(1)) +#define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */ + +/* Some sane limits for resolutions */ +#define IPU3_CSS_MIN_RES 32 +#define IPU3_CSS_MAX_H 3136 +#define IPU3_CSS_MAX_W 4224 + +/* filter size from graph settings is fixed as 4 */ +#define FILTER_SIZE 4 +#define MIN_ENVELOPE 8 + +/* + * pre-allocated buffer size for CSS ABI, auxiliary frames + * after BDS and before GDC. Those values should be tuned + * to big enough to avoid buffer re-allocation when + * streaming to lower streaming latency. + */ +#define CSS_ABI_SIZE 136 +#define CSS_BDS_SIZE (4480 * 3200 * 3) +#define CSS_GDC_SIZE (4224 * 3200 * 12 / 8) + +#define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q)) +#define IPU3_CSS_FORMAT_FL_IN \ + IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN) +#define IPU3_CSS_FORMAT_FL_OUT \ + IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT) +#define IPU3_CSS_FORMAT_FL_VF \ + IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF) + +/* Formats supported by IPU3 Camera Sub System */ +static const struct ipu3_css_format ipu3_css_formats[] = { + { + .pixelformat = V4L2_PIX_FMT_NV12, + .colorspace = V4L2_COLORSPACE_SRGB, + .frame_format = IMGU_ABI_FRAME_FORMAT_NV12, + .osys_format = IMGU_ABI_OSYS_FORMAT_NV12, + .osys_tiling = IMGU_ABI_OSYS_TILING_NONE, + .bytesperpixel_num = 1 * IPU3_CSS_FORMAT_BPP_DEN, + .chroma_decim = 4, + .width_align = IPU3_UAPI_ISP_VEC_ELEMS, + .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF, + }, { + /* Each 32 bytes contains 25 10-bit pixels */ + .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10, + .colorspace = V4L2_COLORSPACE_RAW, + .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, + .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR, + .bit_depth = 10, + .bytesperpixel_num = 64, + .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, + .flags = IPU3_CSS_FORMAT_FL_IN, + }, { + .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10, + .colorspace = V4L2_COLORSPACE_RAW, + .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, + .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG, + .bit_depth = 10, + .bytesperpixel_num = 64, + .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, + .flags = IPU3_CSS_FORMAT_FL_IN, + }, { + .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10, + .colorspace = V4L2_COLORSPACE_RAW, + .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, + .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG, + .bit_depth = 10, + .bytesperpixel_num = 64, + .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, + .flags = IPU3_CSS_FORMAT_FL_IN, + }, { + .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10, + .colorspace = V4L2_COLORSPACE_RAW, + .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, + .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB, + .bit_depth = 10, + .bytesperpixel_num = 64, + .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, + .flags = IPU3_CSS_FORMAT_FL_IN, + }, +}; + +static const struct { + enum imgu_abi_queue_id qid; + size_t ptr_ofs; +} ipu3_css_queues[IPU3_CSS_QUEUES] = { + [IPU3_CSS_QUEUE_IN] = { + IMGU_ABI_QUEUE_C_ID, + offsetof(struct imgu_abi_buffer, payload.frame.frame_data) + }, + [IPU3_CSS_QUEUE_OUT] = { + IMGU_ABI_QUEUE_D_ID, + offsetof(struct imgu_abi_buffer, payload.frame.frame_data) + }, + [IPU3_CSS_QUEUE_VF] = { + IMGU_ABI_QUEUE_E_ID, + offsetof(struct imgu_abi_buffer, payload.frame.frame_data) + }, + [IPU3_CSS_QUEUE_STAT_3A] = { + IMGU_ABI_QUEUE_F_ID, + offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr) + }, +}; + +/* Initialize queue based on given format, adjust format as needed */ +static int ipu3_css_queue_init(struct ipu3_css_queue *queue, + struct v4l2_pix_format_mplane *fmt, u32 flags) +{ + struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix; + unsigned int i; + u32 sizeimage; + + INIT_LIST_HEAD(&queue->bufs); + + queue->css_fmt = NULL; /* Disable */ + if (!fmt) + return 0; + + for (i = 0; i < ARRAY_SIZE(ipu3_css_formats); i++) { + if (!(ipu3_css_formats[i].flags & flags)) + continue; + queue->css_fmt = &ipu3_css_formats[i]; + if (ipu3_css_formats[i].pixelformat == fmt->pixelformat) + break; + } + if (!queue->css_fmt) + return -EINVAL; /* Could not find any suitable format */ + + queue->fmt.mpix = *fmt; + + f->width = ALIGN(clamp_t(u32, f->width, + IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2); + f->height = ALIGN(clamp_t(u32, f->height, + IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2); + queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align); + if (queue->css_fmt->frame_format != IMGU_ABI_FRAME_FORMAT_RAW_PACKED) + f->plane_fmt[0].bytesperline = DIV_ROUND_UP(queue->width_pad * + queue->css_fmt->bytesperpixel_num, + IPU3_CSS_FORMAT_BPP_DEN); + else + /* For packed raw, alignment for bpl is by 50 to the width */ + f->plane_fmt[0].bytesperline = + DIV_ROUND_UP(f->width, + IPU3_CSS_FORMAT_BPP_DEN) * + queue->css_fmt->bytesperpixel_num; + + sizeimage = f->height * f->plane_fmt[0].bytesperline; + if (queue->css_fmt->chroma_decim) + sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim; + + f->plane_fmt[0].sizeimage = sizeimage; + f->field = V4L2_FIELD_NONE; + f->num_planes = 1; + f->colorspace = queue->css_fmt->colorspace; + f->flags = 0; + f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + f->quantization = V4L2_QUANTIZATION_DEFAULT; + f->xfer_func = V4L2_XFER_FUNC_DEFAULT; + memset(f->reserved, 0, sizeof(f->reserved)); + + return 0; +} + +static bool ipu3_css_queue_enabled(struct ipu3_css_queue *q) +{ + return q->css_fmt; +} + /******************* css hw *******************/ /* In the style of writesl() defined in include/asm-generic/io.h */ @@ -492,6 +659,1579 @@ static void ipu3_css_hw_cleanup(struct ipu3_css *css) usleep_range(200, 300); } +static void ipu3_css_pipeline_cleanup(struct ipu3_css *css) +{ + struct imgu_device *imgu = dev_get_drvdata(css->dev); + unsigned int i; + + ipu3_css_pool_cleanup(imgu, &css->pool.parameter_set_info); + ipu3_css_pool_cleanup(imgu, &css->pool.acc); + ipu3_css_pool_cleanup(imgu, &css->pool.gdc); + ipu3_css_pool_cleanup(imgu, &css->pool.obgrid); + + for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) + ipu3_css_pool_cleanup(imgu, &css->pool.binary_params_p[i]); +} + +/* + * This function initializes various stages of the + * IPU3 CSS ISP pipeline + */ +static int ipu3_css_pipeline_init(struct ipu3_css *css) +{ + static const unsigned int PIPE_ID = IPU3_CSS_PIPE_ID_VIDEO; + static const int BYPC = 2; /* Bytes per component */ + static const struct imgu_abi_buffer_sp buffer_sp_init = { + .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID}, + .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID, + }; + + struct imgu_abi_isp_iterator_config *cfg_iter; + struct imgu_abi_isp_ref_config *cfg_ref; + struct imgu_abi_isp_dvs_config *cfg_dvs; + struct imgu_abi_isp_tnr3_config *cfg_tnr; + struct imgu_abi_isp_ref_dmem_state *cfg_ref_state; + struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state; + + const int pipe = 0, stage = 0, thread = 0; + unsigned int i, j; + + const struct imgu_fw_info *bi = + &css->fwp->binary_header[css->current_binary]; + const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes; + + struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp + + bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG]; + struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp + + bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE]; + + struct imgu_abi_isp_stage *isp_stage; + struct imgu_abi_sp_stage *sp_stage; + struct imgu_abi_sp_group *sp_group; + + const unsigned int bds_width_pad = + ALIGN(css->rect[IPU3_CSS_RECT_BDS].width, + 2 * IPU3_UAPI_ISP_VEC_ELEMS); + + const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0; + enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG; + void *vaddr = css->binary_params_cs[cfg - 1][m0].vaddr; + + struct imgu_device *imgu = dev_get_drvdata(css->dev); + + /* Configure iterator */ + + cfg_iter = ipu3_css_fw_pipeline_params(css, cfg, m0, + &cofs->dmem.iterator, + sizeof(*cfg_iter), vaddr); + if (!cfg_iter) + goto bad_firmware; + + cfg_iter->input_info.res.width = + css->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width; + cfg_iter->input_info.res.height = + css->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height; + cfg_iter->input_info.padded_width = + css->queue[IPU3_CSS_QUEUE_IN].width_pad; + cfg_iter->input_info.format = + css->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format; + cfg_iter->input_info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth; + cfg_iter->input_info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order; + cfg_iter->input_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + + cfg_iter->internal_info.res.width = css->rect[IPU3_CSS_RECT_BDS].width; + cfg_iter->internal_info.res.height = + css->rect[IPU3_CSS_RECT_BDS].height; + cfg_iter->internal_info.padded_width = bds_width_pad; + cfg_iter->internal_info.format = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; + cfg_iter->internal_info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; + cfg_iter->internal_info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; + cfg_iter->internal_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + + cfg_iter->output_info.res.width = + css->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width; + cfg_iter->output_info.res.height = + css->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; + cfg_iter->output_info.padded_width = + css->queue[IPU3_CSS_QUEUE_OUT].width_pad; + cfg_iter->output_info.format = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; + cfg_iter->output_info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; + cfg_iter->output_info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; + cfg_iter->output_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + + cfg_iter->vf_info.res.width = + css->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width; + cfg_iter->vf_info.res.height = + css->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; + cfg_iter->vf_info.padded_width = + css->queue[IPU3_CSS_QUEUE_VF].width_pad; + cfg_iter->vf_info.format = + css->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; + cfg_iter->vf_info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth; + cfg_iter->vf_info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order; + cfg_iter->vf_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + + cfg_iter->dvs_envelope.width = css->rect[IPU3_CSS_RECT_ENVELOPE].width; + cfg_iter->dvs_envelope.height = + css->rect[IPU3_CSS_RECT_ENVELOPE].height; + + /* Configure reference (delay) frames */ + + cfg_ref = ipu3_css_fw_pipeline_params(css, cfg, m0, &cofs->dmem.ref, + sizeof(*cfg_ref), vaddr); + if (!cfg_ref) + goto bad_firmware; + + cfg_ref->port_b.crop = 0; + cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC; + cfg_ref->port_b.width = css->aux_frames[IPU3_CSS_AUX_FRAME_REF].width; + cfg_ref->port_b.stride = + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline; + cfg_ref->width_a_over_b = + IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems; + cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) { + cfg_ref->ref_frame_addr_y[i] = + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr; + cfg_ref->ref_frame_addr_c[i] = + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr + + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline * + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; + } + for (; i < IMGU_ABI_FRAMES_REF; i++) { + cfg_ref->ref_frame_addr_y[i] = 0; + cfg_ref->ref_frame_addr_c[i] = 0; + } + + /* Configure DVS (digital video stabilization) */ + + cfg_dvs = ipu3_css_fw_pipeline_params(css, cfg, m0, + &cofs->dmem.dvs, sizeof(*cfg_dvs), + vaddr); + if (!cfg_dvs) + goto bad_firmware; + + cfg_dvs->num_horizontal_blocks = + ALIGN(DIV_ROUND_UP(css->rect[IPU3_CSS_RECT_GDC].width, + IMGU_DVS_BLOCK_W), 2); + cfg_dvs->num_vertical_blocks = + DIV_ROUND_UP(css->rect[IPU3_CSS_RECT_GDC].height, + IMGU_DVS_BLOCK_H); + + /* Configure TNR (temporal noise reduction) */ + + if (css->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { + cfg_tnr = ipu3_css_fw_pipeline_params(css, cfg, m0, + &cofs->dmem.tnr3, + sizeof(*cfg_tnr), + vaddr); + if (!cfg_tnr) + goto bad_firmware; + + cfg_tnr->port_b.crop = 0; + cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES; + cfg_tnr->port_b.width = + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; + cfg_tnr->port_b.stride = + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline; + cfg_tnr->width_a_over_b = + IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems; + cfg_tnr->frame_height = + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; + cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1; + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + cfg_tnr->frame_addr[i] = + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR] + .mem[i].daddr; + for (; i < IMGU_ABI_FRAMES_TNR; i++) + cfg_tnr->frame_addr[i] = 0; + } + + /* Configure ref dmem state parameters */ + + cfg = IMGU_ABI_PARAM_CLASS_STATE; + vaddr = css->binary_params_cs[cfg - 1][m0].vaddr; + + cfg_ref_state = ipu3_css_fw_pipeline_params(css, cfg, m0, + &sofs->dmem.ref, + sizeof(*cfg_ref_state), + vaddr); + if (!cfg_ref_state) + goto bad_firmware; + + cfg_ref_state->ref_in_buf_idx = 0; + cfg_ref_state->ref_out_buf_idx = 1; + + /* Configure tnr dmem state parameters */ + if (css->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { + cfg_tnr_state = + ipu3_css_fw_pipeline_params(css, cfg, m0, + &sofs->dmem.tnr3, + sizeof(*cfg_tnr_state), + vaddr); + if (!cfg_tnr_state) + goto bad_firmware; + + cfg_tnr_state->in_bufidx = 0; + cfg_tnr_state->out_bufidx = 1; + cfg_tnr_state->bypass_filter = 0; + cfg_tnr_state->total_frame_counter = 0; + for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++) + cfg_tnr_state->buffer_frame_counter[i] = 0; + } + + /* Configure ISP stage */ + + isp_stage = css->xmem_isp_stage_ptrs[pipe][stage].vaddr; + memset(isp_stage, 0, sizeof(*isp_stage)); + isp_stage->blob_info = bi->blob; + isp_stage->binary_info = bi->info.isp.sp; + strscpy(isp_stage->binary_name, + (char *)css->fwp + bi->blob.prog_name_offset, + sizeof(isp_stage->binary_name)); + isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers; + for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++) + for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++) + isp_stage->mem_initializers.params[i][j].address = + css->binary_params_cs[i - 1][j].daddr; + + /* Configure SP stage */ + + sp_stage = css->xmem_sp_stage_ptrs[pipe][stage].vaddr; + memset(sp_stage, 0, sizeof(*sp_stage)); + + sp_stage->frames.in.buf_attr = buffer_sp_init; + for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++) + sp_stage->frames.out[i].buf_attr = buffer_sp_init; + sp_stage->frames.out_vf.buf_attr = buffer_sp_init; + sp_stage->frames.s3a_buf = buffer_sp_init; + sp_stage->frames.dvs_buf = buffer_sp_init; + + sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP; + sp_stage->num = stage; + sp_stage->isp_online = 0; + sp_stage->isp_copy_vf = 0; + sp_stage->isp_copy_output = 0; + + /* Enable VF output only when VF or PV queue requested by user */ + + sp_stage->enable.vf_output = + (css->vf_output_en != IPU3_NODE_VF_DISABLED); + + sp_stage->frames.effective_in_res.width = + css->rect[IPU3_CSS_RECT_EFFECTIVE].width; + sp_stage->frames.effective_in_res.height = + css->rect[IPU3_CSS_RECT_EFFECTIVE].height; + sp_stage->frames.in.info.res.width = + css->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width; + sp_stage->frames.in.info.res.height = + css->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height; + sp_stage->frames.in.info.padded_width = + css->queue[IPU3_CSS_QUEUE_IN].width_pad; + sp_stage->frames.in.info.format = + css->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format; + sp_stage->frames.in.info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth; + sp_stage->frames.in.info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order; + sp_stage->frames.in.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + sp_stage->frames.in.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID; + sp_stage->frames.in.buf_attr.buf_type = + IMGU_ABI_BUFFER_TYPE_INPUT_FRAME; + + sp_stage->frames.out[0].info.res.width = + css->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width; + sp_stage->frames.out[0].info.res.height = + css->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; + sp_stage->frames.out[0].info.padded_width = + css->queue[IPU3_CSS_QUEUE_OUT].width_pad; + sp_stage->frames.out[0].info.format = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; + sp_stage->frames.out[0].info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; + sp_stage->frames.out[0].info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; + sp_stage->frames.out[0].info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + sp_stage->frames.out[0].planes.nv.uv.offset = + css->queue[IPU3_CSS_QUEUE_OUT].width_pad * + css->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; + sp_stage->frames.out[0].buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID; + sp_stage->frames.out[0].buf_attr.buf_type = + IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME; + + sp_stage->frames.out[1].buf_attr.buf_src.queue_id = + IMGU_ABI_QUEUE_EVENT_ID; + + sp_stage->frames.internal_frame_info.res.width = + css->rect[IPU3_CSS_RECT_BDS].width; + sp_stage->frames.internal_frame_info.res.height = + css->rect[IPU3_CSS_RECT_BDS].height; + sp_stage->frames.internal_frame_info.padded_width = bds_width_pad; + + sp_stage->frames.internal_frame_info.format = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; + sp_stage->frames.internal_frame_info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; + sp_stage->frames.internal_frame_info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; + sp_stage->frames.internal_frame_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + + sp_stage->frames.out_vf.info.res.width = + css->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width; + sp_stage->frames.out_vf.info.res.height = + css->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; + sp_stage->frames.out_vf.info.padded_width = + css->queue[IPU3_CSS_QUEUE_VF].width_pad; + sp_stage->frames.out_vf.info.format = + css->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; + sp_stage->frames.out_vf.info.raw_bit_depth = + css->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth; + sp_stage->frames.out_vf.info.raw_bayer_order = + css->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order; + sp_stage->frames.out_vf.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; + sp_stage->frames.out_vf.planes.yuv.u.offset = + css->queue[IPU3_CSS_QUEUE_VF].width_pad * + css->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; + sp_stage->frames.out_vf.planes.yuv.v.offset = + css->queue[IPU3_CSS_QUEUE_VF].width_pad * + css->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height * 5 / 4; + sp_stage->frames.out_vf.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID; + sp_stage->frames.out_vf.buf_attr.buf_type = + IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME; + + sp_stage->frames.s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID; + sp_stage->frames.s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS; + + sp_stage->frames.dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID; + sp_stage->frames.dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS; + + sp_stage->dvs_envelope.width = css->rect[IPU3_CSS_RECT_ENVELOPE].width; + sp_stage->dvs_envelope.height = + css->rect[IPU3_CSS_RECT_ENVELOPE].height; + + sp_stage->isp_pipe_version = + bi->info.isp.sp.pipeline.isp_pipe_version; + sp_stage->isp_deci_log_factor = + clamp(max(fls(css->rect[IPU3_CSS_RECT_BDS].width / + IMGU_MAX_BQ_GRID_WIDTH), + fls(css->rect[IPU3_CSS_RECT_BDS].height / + IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5); + sp_stage->isp_vf_downscale_bits = 0; + sp_stage->if_config_index = 255; + sp_stage->sp_enable_xnr = 0; + sp_stage->num_stripes = stripes; + sp_stage->enable.s3a = 1; + sp_stage->enable.dvs_stats = 0; + + sp_stage->xmem_bin_addr = css->binary[css->current_binary].daddr; + sp_stage->xmem_map_addr = css->sp_ddr_ptrs.daddr; + sp_stage->isp_stage_addr = css->xmem_isp_stage_ptrs[pipe][stage].daddr; + + /* Configure SP group */ + + sp_group = css->xmem_sp_group_ptrs.vaddr; + memset(sp_group, 0, sizeof(*sp_group)); + + sp_group->pipe[thread].num_stages = 1; + sp_group->pipe[thread].pipe_id = PIPE_ID; + sp_group->pipe[thread].thread_id = thread; + sp_group->pipe[thread].pipe_num = pipe; + sp_group->pipe[thread].num_execs = -1; + sp_group->pipe[thread].pipe_qos_config = -1; + sp_group->pipe[thread].required_bds_factor = 0; + sp_group->pipe[thread].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; + sp_group->pipe[thread].inout_port_config = + IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST | + IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST; + sp_group->pipe[thread].scaler_pp_lut = 0; + sp_group->pipe[thread].shading.internal_frame_origin_x_bqs_on_sctbl = 0; + sp_group->pipe[thread].shading.internal_frame_origin_y_bqs_on_sctbl = 0; + sp_group->pipe[thread].sp_stage_addr[stage] = + css->xmem_sp_stage_ptrs[pipe][stage].daddr; + sp_group->pipe[thread].pipe_config = + bi->info.isp.sp.enable.params ? (1 << thread) : 0; + sp_group->pipe[thread].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP; + + /* Initialize parameter pools */ + + if (ipu3_css_pool_init(imgu, &css->pool.parameter_set_info, + sizeof(struct imgu_abi_parameter_set_info)) || + ipu3_css_pool_init(imgu, &css->pool.acc, + sizeof(struct imgu_abi_acc_param)) || + ipu3_css_pool_init(imgu, &css->pool.gdc, + sizeof(struct imgu_abi_gdc_warp_param) * + 3 * cfg_dvs->num_horizontal_blocks / 2 * + cfg_dvs->num_vertical_blocks) || + ipu3_css_pool_init(imgu, &css->pool.obgrid, + ipu3_css_fw_obgrid_size( + &css->fwp->binary_header[css->current_binary]))) + goto out_of_memory; + + for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) + if (ipu3_css_pool_init(imgu, &css->pool.binary_params_p[i], + bi->info.isp.sp.mem_initializers.params + [IMGU_ABI_PARAM_CLASS_PARAM][i].size)) + goto out_of_memory; + + return 0; + +bad_firmware: + ipu3_css_pipeline_cleanup(css); + return -EPROTO; + +out_of_memory: + ipu3_css_pipeline_cleanup(css); + return -ENOMEM; +} + +static u8 ipu3_css_queue_pos(struct ipu3_css *css, int queue, int thread) +{ + static const unsigned int sp; + void __iomem *const base = css->base; + struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; + struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + + bi->info.sp.host_sp_queue; + + return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) : + readb(&q->host2sp_evtq_info.end); +} + +/* Sent data to sp using given buffer queue, or if queue < 0, event queue. */ +static int ipu3_css_queue_data(struct ipu3_css *css, + int queue, int thread, u32 data) +{ + static const unsigned int sp; + void __iomem *const base = css->base; + struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; + struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + + bi->info.sp.host_sp_queue; + u8 size, start, end, end2; + + if (queue >= 0) { + size = readb(&q->host2sp_bufq_info[thread][queue].size); + start = readb(&q->host2sp_bufq_info[thread][queue].start); + end = readb(&q->host2sp_bufq_info[thread][queue].end); + } else { + size = readb(&q->host2sp_evtq_info.size); + start = readb(&q->host2sp_evtq_info.start); + end = readb(&q->host2sp_evtq_info.end); + } + + if (size == 0) + return -EIO; + + end2 = (end + 1) % size; + if (end2 == start) + return -EBUSY; /* Queue full */ + + if (queue >= 0) { + writel(data, &q->host2sp_bufq[thread][queue][end]); + writeb(end2, &q->host2sp_bufq_info[thread][queue].end); + } else { + writel(data, &q->host2sp_evtq[end]); + writeb(end2, &q->host2sp_evtq_info.end); + } + + return 0; +} + +/* Receive data using given buffer queue, or if queue < 0, event queue. */ +static int ipu3_css_dequeue_data(struct ipu3_css *css, int queue, u32 *data) +{ + static const unsigned int sp; + void __iomem *const base = css->base; + struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; + struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + + bi->info.sp.host_sp_queue; + u8 size, start, end, start2; + + if (queue >= 0) { + size = readb(&q->sp2host_bufq_info[queue].size); + start = readb(&q->sp2host_bufq_info[queue].start); + end = readb(&q->sp2host_bufq_info[queue].end); + } else { + size = readb(&q->sp2host_evtq_info.size); + start = readb(&q->sp2host_evtq_info.start); + end = readb(&q->sp2host_evtq_info.end); + } + + if (size == 0) + return -EIO; + + if (end == start) + return -EBUSY; /* Queue empty */ + + start2 = (start + 1) % size; + + if (queue >= 0) { + *data = readl(&q->sp2host_bufq[queue][start]); + writeb(start2, &q->sp2host_bufq_info[queue].start); + } else { + int r; + + *data = readl(&q->sp2host_evtq[start]); + writeb(start2, &q->sp2host_evtq_info.start); + + /* Acknowledge events dequeued from event queue */ + r = ipu3_css_queue_data(css, queue, 0, + IMGU_ABI_EVENT_EVENT_DEQUEUED); + if (r < 0) + return r; + } + + return 0; +} + +/* Free binary-specific resources */ +static void ipu3_css_binary_cleanup(struct ipu3_css *css) +{ + struct imgu_device *imgu = dev_get_drvdata(css->dev); + unsigned int i, j; + + for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++) + for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) + ipu3_dmamap_free(imgu, &css->binary_params_cs[j][i]); + + j = IPU3_CSS_AUX_FRAME_REF; + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + ipu3_dmamap_free(imgu, &css->aux_frames[j].mem[i]); + + j = IPU3_CSS_AUX_FRAME_TNR; + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + ipu3_dmamap_free(imgu, &css->aux_frames[j].mem[i]); +} + +static int ipu3_css_binary_preallocate(struct ipu3_css *css) +{ + struct imgu_device *imgu = dev_get_drvdata(css->dev); + unsigned int i, j; + + for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++) + for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) { + if (!ipu3_dmamap_alloc(imgu, + &css->binary_params_cs[j - 1][i], + CSS_ABI_SIZE)) + goto out_of_memory; + } + + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + if (!ipu3_dmamap_alloc(imgu, + &css->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], + CSS_BDS_SIZE)) + goto out_of_memory; + + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + if (!ipu3_dmamap_alloc(imgu, + &css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], + CSS_GDC_SIZE)) + goto out_of_memory; + + return 0; + +out_of_memory: + ipu3_css_binary_cleanup(css); + return -ENOMEM; +} + +/* allocate binary-specific resources */ +static int ipu3_css_binary_setup(struct ipu3_css *css) +{ + const struct imgu_abi_binary_info *sp = + &css->fwp->binary_header[css->current_binary].info.isp.sp; + struct imgu_device *imgu = dev_get_drvdata(css->dev); + static const int BYPC = 2; /* Bytes per component */ + unsigned int w, h, size, i, j; + + /* Allocate parameter memory blocks for this binary */ + + for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++) + for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) { + if (ipu3_css_dma_buffer_resize( + imgu, &css->binary_params_cs[j - 1][i], + sp->mem_initializers.params[j][i].size)) + goto out_of_memory; + } + + /* Allocate internal frame buffers */ + + /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */ + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC; + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].width = + css->rect[IPU3_CSS_RECT_BDS].width; + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].height = + ALIGN(css->rect[IPU3_CSS_RECT_BDS].height, + IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y; + h = css->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; + w = ALIGN(css->rect[IPU3_CSS_RECT_BDS].width, + 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X; + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline = + css->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w; + size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2; + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + if (ipu3_css_dma_buffer_resize( + imgu, + &css->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], size)) + goto out_of_memory; + + /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */ + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1; + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width = + roundup(css->rect[IPU3_CSS_RECT_GDC].width, + sp->block.block_width * + IPU3_UAPI_ISP_VEC_ELEMS); + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height = + roundup(css->rect[IPU3_CSS_RECT_GDC].height, + sp->block.output_block_height); + + w = css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; + css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w; + h = css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; + size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */ + for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) + if (ipu3_css_dma_buffer_resize( + imgu, + &css->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], size)) + goto out_of_memory; + + return 0; + +out_of_memory: + ipu3_css_binary_cleanup(css); + return -ENOMEM; +} + +int ipu3_css_start_streaming(struct ipu3_css *css) +{ + u32 data; + int r; + + if (css->streaming) + return -EPROTO; + + r = ipu3_css_binary_setup(css); + if (r < 0) + return r; + + r = ipu3_css_hw_init(css); + if (r < 0) + return r; + + r = ipu3_css_hw_start(css); + if (r < 0) + goto fail; + + r = ipu3_css_pipeline_init(css); + if (r < 0) + goto fail; + + css->streaming = true; + + ipu3_css_hw_enable_irq(css); + + /* Initialize parameters to default */ + r = ipu3_css_set_parameters(css, NULL); + if (r < 0) + goto fail; + + while (!(r = ipu3_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data))) + ; + if (r != -EBUSY) + goto fail; + + while (!(r = ipu3_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data))) + ; + if (r != -EBUSY) + goto fail; + + r = ipu3_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, 0, + IMGU_ABI_EVENT_START_STREAM); + if (r < 0) + goto fail; + + return 0; + +fail: + css->streaming = false; + ipu3_css_hw_cleanup(css); + ipu3_css_pipeline_cleanup(css); + ipu3_css_binary_cleanup(css); + + return r; +} + +void ipu3_css_stop_streaming(struct ipu3_css *css) +{ + struct ipu3_css_buffer *b, *b0; + int q, r; + + r = ipu3_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, 0, + IMGU_ABI_EVENT_STOP_STREAM); + + if (r < 0) + dev_warn(css->dev, "failed on stop stream event\n"); + + if (!css->streaming) + return; + + ipu3_css_hw_stop(css); + + ipu3_css_hw_cleanup(css); + + ipu3_css_pipeline_cleanup(css); + + spin_lock(&css->qlock); + for (q = 0; q < IPU3_CSS_QUEUES; q++) + list_for_each_entry_safe(b, b0, &css->queue[q].bufs, list) { + b->state = IPU3_CSS_BUFFER_FAILED; + list_del(&b->list); + } + spin_unlock(&css->qlock); + + css->streaming = false; +} + +bool ipu3_css_queue_empty(struct ipu3_css *css) +{ + int q; + + spin_lock(&css->qlock); + for (q = 0; q < IPU3_CSS_QUEUES; q++) + if (!list_empty(&css->queue[q].bufs)) + break; + spin_unlock(&css->qlock); + + return (q == IPU3_CSS_QUEUES); +} + +bool ipu3_css_is_streaming(struct ipu3_css *css) +{ + return css->streaming; +} + +void ipu3_css_cleanup(struct ipu3_css *css) +{ + struct imgu_device *imgu = dev_get_drvdata(css->dev); + unsigned int p, q, i; + + ipu3_css_stop_streaming(css); + ipu3_css_binary_cleanup(css); + + for (q = 0; q < IPU3_CSS_QUEUES; q++) + for (i = 0; i < ARRAY_SIZE(css->abi_buffers[q]); i++) + ipu3_dmamap_free(imgu, &css->abi_buffers[q][i]); + + for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) + for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { + ipu3_dmamap_free(imgu, &css->xmem_sp_stage_ptrs[p][i]); + ipu3_dmamap_free(imgu, &css->xmem_isp_stage_ptrs[p][i]); + } + + ipu3_dmamap_free(imgu, &css->sp_ddr_ptrs); + ipu3_dmamap_free(imgu, &css->xmem_sp_group_ptrs); + + ipu3_css_fw_cleanup(css); +} + +int ipu3_css_init(struct device *dev, struct ipu3_css *css, + void __iomem *base, int length) +{ + struct imgu_device *imgu = dev_get_drvdata(dev); + int r, p, q, i; + + /* Initialize main data structure */ + css->dev = dev; + css->base = base; + css->iomem_length = length; + css->current_binary = IPU3_CSS_DEFAULT_BINARY; + css->pipe_id = IPU3_CSS_PIPE_ID_NUM; + css->vf_output_en = IPU3_NODE_VF_DISABLED; + spin_lock_init(&css->qlock); + + for (q = 0; q < IPU3_CSS_QUEUES; q++) { + r = ipu3_css_queue_init(&css->queue[q], NULL, 0); + if (r) + return r; + } + + r = ipu3_css_fw_init(css); + if (r) + return r; + + /* Allocate and map common structures with imgu hardware */ + + for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) + for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { + if (!ipu3_dmamap_alloc(imgu, + &css->xmem_sp_stage_ptrs[p][i], + sizeof(struct imgu_abi_sp_stage))) + goto error_no_memory; + if (!ipu3_dmamap_alloc(imgu, + &css->xmem_isp_stage_ptrs[p][i], + sizeof(struct imgu_abi_isp_stage))) + goto error_no_memory; + } + + if (!ipu3_dmamap_alloc(imgu, &css->sp_ddr_ptrs, + ALIGN(sizeof(struct imgu_abi_ddr_address_map), + IMGU_ABI_ISP_DDR_WORD_BYTES))) + goto error_no_memory; + + if (!ipu3_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs, + sizeof(struct imgu_abi_sp_group))) + goto error_no_memory; + + for (q = 0; q < IPU3_CSS_QUEUES; q++) + for (i = 0; i < ARRAY_SIZE(css->abi_buffers[q]); i++) + if (!ipu3_dmamap_alloc(imgu, &css->abi_buffers[q][i], + sizeof(struct imgu_abi_buffer))) + goto error_no_memory; + + if (ipu3_css_binary_preallocate(css)) + goto error_binary_setup; + + return 0; + +error_binary_setup: + ipu3_css_binary_cleanup(css); +error_no_memory: + ipu3_css_cleanup(css); + + return -ENOMEM; +} + +static u32 ipu3_css_adjust(u32 res, u32 align) +{ + u32 val = max_t(u32, IPU3_CSS_MIN_RES, res); + + return DIV_ROUND_CLOSEST(val, align) * align; +} + +/* Select a binary matching the required resolutions and formats */ +static int ipu3_css_find_binary(struct ipu3_css *css, + struct ipu3_css_queue queue[IPU3_CSS_QUEUES], + struct v4l2_rect rects[IPU3_CSS_RECTS]) +{ + const int binary_nr = css->fwp->file_header.binary_nr; + unsigned int binary_mode = (css->pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ? + IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO; + const struct v4l2_pix_format_mplane *in = + &queue[IPU3_CSS_QUEUE_IN].fmt.mpix; + const struct v4l2_pix_format_mplane *out = + &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix; + const struct v4l2_pix_format_mplane *vf = + &queue[IPU3_CSS_QUEUE_VF].fmt.mpix; + u32 stripe_w = 0, stripe_h = 0; + const char *name; + int i, j; + + if (!ipu3_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN])) + return -EINVAL; + + /* Find out the strip size boundary */ + for (i = 0; i < binary_nr; i++) { + struct imgu_fw_info *bi = &css->fwp->binary_header[i]; + + u32 max_width = bi->info.isp.sp.output.max_width; + u32 max_height = bi->info.isp.sp.output.max_height; + + if (bi->info.isp.sp |