drivers/staging/media/ipu3/ipu3-css.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2018 Intel Corporation
   3
   4 #include <linux/device.h>
   5 #include <linux/iopoll.h>
   6 #include <linux/slab.h>
   7
   8 #include "ipu3.h"
   9 #include "ipu3-css.h"
  10 #include "ipu3-css-fw.h"
  11 #include "ipu3-css-params.h"
  12 #include "ipu3-dmamap.h"
  13 #include "ipu3-tables.h"
  14
  15 /* IRQ configuration */
  16 #define IMGU_IRQCTRL_IRQ_MASK   (IMGU_IRQCTRL_IRQ_SP1 | \
  17                                  IMGU_IRQCTRL_IRQ_SP2 | \
  18                                  IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
  19                                  IMGU_IRQCTRL_IRQ_SW_PIN(1))
  20
  21 #define IPU3_CSS_FORMAT_BPP_DEN 50      /* Denominator */
  22
  23 /* Some sane limits for resolutions */
  24 #define IPU3_CSS_MIN_RES        32
  25 #define IPU3_CSS_MAX_H          3136
  26 #define IPU3_CSS_MAX_W          4224
  27
  28 /* minimal envelope size(GDC in - out) should be 4 */
  29 #define MIN_ENVELOPE            4
  30
  31 /*
  32  * pre-allocated buffer size for CSS ABI, auxiliary frames
  33  * after BDS and before GDC. Those values should be tuned
  34  * to big enough to avoid buffer re-allocation when
  35  * streaming to lower streaming latency.
  36  */
  37 #define CSS_ABI_SIZE    136
  38 #define CSS_BDS_SIZE    (4480 * 3200 * 3)
  39 #define CSS_GDC_SIZE    (4224 * 3200 * 12 / 8)
  40
  41 #define IPU3_CSS_QUEUE_TO_FLAGS(q)      (1 << (q))
  42 #define IPU3_CSS_FORMAT_FL_IN           \
  43                         IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
  44 #define IPU3_CSS_FORMAT_FL_OUT          \
  45                         IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
  46 #define IPU3_CSS_FORMAT_FL_VF           \
  47                         IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
  48
  49 /* Formats supported by IPU3 Camera Sub System */
  50 static const struct imgu_css_format imgu_css_formats[] = {
  51         {
  52                 .pixelformat = V4L2_PIX_FMT_NV12,
  53                 .colorspace = V4L2_COLORSPACE_SRGB,
  54                 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
  55                 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
  56                 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
  57                 .chroma_decim = 4,
  58                 .width_align = IPU3_UAPI_ISP_VEC_ELEMS,
  59                 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
  60         }, {
  61                 /* Each 32 bytes contains 25 10-bit pixels */
  62                 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
  63                 .colorspace = V4L2_COLORSPACE_RAW,
  64                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  65                 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
  66                 .bit_depth = 10,
  67                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  68                 .flags = IPU3_CSS_FORMAT_FL_IN,
  69         }, {
  70                 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
  71                 .colorspace = V4L2_COLORSPACE_RAW,
  72                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  73                 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
  74                 .bit_depth = 10,
  75                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  76                 .flags = IPU3_CSS_FORMAT_FL_IN,
  77         }, {
  78                 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
  79                 .colorspace = V4L2_COLORSPACE_RAW,
  80                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  81                 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
  82                 .bit_depth = 10,
  83                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  84                 .flags = IPU3_CSS_FORMAT_FL_IN,
  85         }, {
  86                 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
  87                 .colorspace = V4L2_COLORSPACE_RAW,
  88                 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  89                 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
  90                 .bit_depth = 10,
  91                 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  92                 .flags = IPU3_CSS_FORMAT_FL_IN,
  93         },
  94 };
  95
  96 static const struct {
  97         enum imgu_abi_queue_id qid;
  98         size_t ptr_ofs;
  99 } imgu_css_queues[IPU3_CSS_QUEUES] = {
 100         [IPU3_CSS_QUEUE_IN] = {
 101                 IMGU_ABI_QUEUE_C_ID,
 102                 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 103         },
 104         [IPU3_CSS_QUEUE_OUT] = {
 105                 IMGU_ABI_QUEUE_D_ID,
 106                 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 107         },
 108         [IPU3_CSS_QUEUE_VF] = {
 109                 IMGU_ABI_QUEUE_E_ID,
 110                 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 111         },
 112         [IPU3_CSS_QUEUE_STAT_3A] = {
 113                 IMGU_ABI_QUEUE_F_ID,
 114                 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
 115         },
 116 };
 117
 118 /* Initialize queue based on given format, adjust format as needed */
 119 static int imgu_css_queue_init(struct imgu_css_queue *queue,
 120                                struct v4l2_pix_format_mplane *fmt, u32 flags)
 121 {
 122         struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
 123         unsigned int i;
 124         u32 sizeimage;
 125
 126         INIT_LIST_HEAD(&queue->bufs);
 127
 128         queue->css_fmt = NULL;  /* Disable */
 129         if (!fmt)
 130                 return 0;
 131
 132         for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
 133                 if (!(imgu_css_formats[i].flags & flags))
 134                         continue;
 135                 queue->css_fmt = &imgu_css_formats[i];
 136                 if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
 137                         break;
 138         }
 139         if (!queue->css_fmt)
 140                 return -EINVAL; /* Could not find any suitable format */
 141
 142         queue->fmt.mpix = *fmt;
 143
 144         f->width = ALIGN(clamp_t(u32, f->width,
 145                                  IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
 146         f->height = ALIGN(clamp_t(u32, f->height,
 147                                   IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
 148         queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
 149         f->plane_fmt[0].bytesperline =
 150                 imgu_bytesperline(f->width, queue->css_fmt->frame_format);
 151         sizeimage = f->height * f->plane_fmt[0].bytesperline;
 152         if (queue->css_fmt->chroma_decim)
 153                 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
 154
 155         f->plane_fmt[0].sizeimage = sizeimage;
 156         f->field = V4L2_FIELD_NONE;
 157         f->num_planes = 1;
 158         f->colorspace = queue->css_fmt->colorspace;
 159         f->flags = 0;
 160         f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
 161         f->quantization = V4L2_QUANTIZATION_DEFAULT;
 162         f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
 163         memset(f->reserved, 0, sizeof(f->reserved));
 164
 165         return 0;
 166 }
 167
 168 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
 169 {
 170         return q->css_fmt;
 171 }
 172
 173 /******************* css hw *******************/
 174
 175 /* In the style of writesl() defined in include/asm-generic/io.h */
 176 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
 177 {
 178         if (count >= 4) {
 179                 const u32 *buf = mem;
 180
 181                 count /= 4;
 182                 do {
 183                         writel(*buf++, addr);
 184                         addr += 4;
 185                 } while (--count);
 186         }
 187 }
 188
 189 /* Wait until register `reg', masked with `mask', becomes `cmp' */
 190 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
 191 {
 192         u32 val;
 193
 194         return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
 195                                   1000, 100 * 1000);
 196 }
 197
 198 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
 199
 200 int imgu_css_set_powerup(struct device *dev, void __iomem *base,
 201                          unsigned int freq)
 202 {
 203         u32 pm_ctrl, state, val;
 204
 205         dev_dbg(dev, "%s with freq %u\n", __func__, freq);
 206         /* Clear the CSS busy signal */
 207         readl(base + IMGU_REG_GP_BUSY);
 208         writel(0, base + IMGU_REG_GP_BUSY);
 209
 210         /* Wait for idle signal */
 211         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 212                          IMGU_STATE_IDLE_STS)) {
 213                 dev_err(dev, "failed to set CSS idle\n");
 214                 goto fail;
 215         }
 216
 217         /* Reset the css */
 218         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
 219                base + IMGU_REG_PM_CTRL);
 220
 221         usleep_range(200, 300);
 222
 223         /** Prepare CSS */
 224
 225         pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
 226         state = readl(base + IMGU_REG_STATE);
 227
 228         dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
 229                 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
 230
 231         /* Power up CSS using wrapper */
 232         if (state & IMGU_STATE_POWER_DOWN) {
 233                 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
 234                        base + IMGU_REG_PM_CTRL);
 235                 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
 236                                  IMGU_PM_CTRL_START, 0)) {
 237                         dev_err(dev, "failed to power up CSS\n");
 238                         goto fail;
 239                 }
 240                 usleep_range(2000, 3000);
 241         } else {
 242                 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
 243         }
 244
 245         /* Set the busy bit */
 246         writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
 247
 248         /* Set CSS clock frequency */
 249         pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
 250         val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 251         writel(val, base + IMGU_REG_PM_CTRL);
 252         writel(0, base + IMGU_REG_GP_BUSY);
 253         if (imgu_hw_wait(base, IMGU_REG_STATE,
 254                          IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
 255                 dev_err(dev, "failed to pwrdn CSS\n");
 256                 goto fail;
 257         }
 258         val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
 259         writel(val, base + IMGU_REG_SYSTEM_REQ);
 260         writel(1, base + IMGU_REG_GP_BUSY);
 261         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
 262                base + IMGU_REG_PM_CTRL);
 263         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
 264                          IMGU_STATE_HALT_STS)) {
 265                 dev_err(dev, "failed to halt CSS\n");
 266                 goto fail;
 267         }
 268
 269         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
 270                base + IMGU_REG_PM_CTRL);
 271         if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
 272                 dev_err(dev, "failed to start CSS\n");
 273                 goto fail;
 274         }
 275         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
 276                base + IMGU_REG_PM_CTRL);
 277
 278         val = readl(base + IMGU_REG_PM_CTRL);   /* get pm_ctrl */
 279         val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 280         val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 281         writel(val, base + IMGU_REG_PM_CTRL);
 282
 283         return 0;
 284
 285 fail:
 286         imgu_css_set_powerdown(dev, base);
 287         return -EIO;
 288 }
 289
 290 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
 291 {
 292         dev_dbg(dev, "%s\n", __func__);
 293         /* wait for cio idle signal */
 294         if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
 295                          IMGU_CIO_GATE_BURST_MASK, 0))
 296                 dev_warn(dev, "wait cio gate idle timeout");
 297
 298         /* wait for css idle signal */
 299         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 300                          IMGU_STATE_IDLE_STS))
 301                 dev_warn(dev, "wait css idle timeout\n");
 302
 303         /* do halt-halted handshake with css */
 304         writel(1, base + IMGU_REG_GP_HALT);
 305         if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
 306                          IMGU_STATE_HALT_STS))
 307                 dev_warn(dev, "failed to halt css");
 308
 309         /* de-assert the busy bit */
 310         writel(0, base + IMGU_REG_GP_BUSY);
 311 }
 312
 313 static void imgu_css_hw_enable_irq(struct imgu_css *css)
 314 {
 315         void __iomem *const base = css->base;
 316         u32 val, i;
 317
 318         /* Set up interrupts */
 319
 320         /*
 321          * Enable IRQ on the SP which signals that SP goes to idle
 322          * (aka ready state) and set trigger to pulse
 323          */
 324         val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
 325         writel(val, base + IMGU_REG_SP_CTRL(0));
 326         writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
 327
 328         /* Enable IRQs from the IMGU wrapper */
 329         writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
 330         /* Clear */
 331         writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
 332
 333         /* Enable IRQs from main IRQ controller */
 334         writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
 335         writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
 336         writel(IMGU_IRQCTRL_IRQ_MASK,
 337                base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
 338         writel(IMGU_IRQCTRL_IRQ_MASK,
 339                base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
 340         writel(IMGU_IRQCTRL_IRQ_MASK,
 341                base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
 342         writel(IMGU_IRQCTRL_IRQ_MASK,
 343                base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
 344         /* Wait for write complete */
 345         readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
 346
 347         /* Enable IRQs from SP0 and SP1 controllers */
 348         for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
 349                 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
 350                 writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
 351                 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
 352                 writel(IMGU_IRQCTRL_IRQ_MASK,
 353                        base + IMGU_REG_IRQCTRL_ENABLE(i));
 354                 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
 355                 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
 356                 /* Wait for write complete */
 357                 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
 358         }
 359 }
 360
 361 static int imgu_css_hw_init(struct imgu_css *css)
 362 {
 363         /* For checking that streaming monitor statuses are valid */
 364         static const struct {
 365                 u32 reg;
 366                 u32 mask;
 367                 const char *name;
 368         } stream_monitors[] = {
 369                 {
 370                         IMGU_REG_GP_SP1_STRMON_STAT,
 371                         IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
 372                         "ISP0 to SP0"
 373                 }, {
 374                         IMGU_REG_GP_ISP_STRMON_STAT,
 375                         IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
 376                         "SP0 to ISP0"
 377                 }, {
 378                         IMGU_REG_GP_MOD_STRMON_STAT,
 379                         IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
 380                         "ISP0 to DMA0"
 381                 }, {
 382                         IMGU_REG_GP_ISP_STRMON_STAT,
 383                         IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
 384                         "DMA0 to ISP0"
 385                 }, {
 386                         IMGU_REG_GP_MOD_STRMON_STAT,
 387                         IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
 388                         "ISP0 to GDC0"
 389                 }, {
 390                         IMGU_REG_GP_MOD_STRMON_STAT,
 391                         IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
 392                         "GDC0 to ISP0"
 393                 }, {
 394                         IMGU_REG_GP_MOD_STRMON_STAT,
 395                         IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
 396                         "SP0 to DMA0"
 397                 }, {
 398                         IMGU_REG_GP_SP1_STRMON_STAT,
 399                         IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
 400                         "DMA0 to SP0"
 401                 }, {
 402                         IMGU_REG_GP_MOD_STRMON_STAT,
 403                         IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
 404                         "SP0 to GDC0"
 405                 }, {
 406                         IMGU_REG_GP_MOD_STRMON_STAT,
 407                         IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
 408                         "GDC0 to SP0"
 409                 },
 410         };
 411
 412         struct device *dev = css->dev;
 413         void __iomem *const base = css->base;
 414         u32 val, i;
 415
 416         /* Set instruction cache address and inv bit for ISP, SP, and SP1 */
 417         for (i = 0; i < IMGU_NUM_SP; i++) {
 418                 struct imgu_fw_info *bi =
 419                                         &css->fwp->binary_header[css->fw_sp[i]];
 420
 421                 writel(css->binary[css->fw_sp[i]].daddr,
 422                        base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
 423                 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
 424                        IMGU_CTRL_ICACHE_INV,
 425                        base + IMGU_REG_SP_CTRL(bi->type));
 426         }
 427         writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
 428         writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
 429                base + IMGU_REG_ISP_CTRL);
 430
 431         /* Check that IMGU hardware is ready */
 432
 433         if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
 434                 dev_err(dev, "SP is not idle\n");
 435                 return -EIO;
 436         }
 437         if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
 438                 dev_err(dev, "ISP is not idle\n");
 439                 return -EIO;
 440         }
 441
 442         for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
 443                 val = readl(base + stream_monitors[i].reg);
 444                 if (val & stream_monitors[i].mask) {
 445                         dev_err(dev, "error: Stream monitor %s is valid\n",
 446                                 stream_monitors[i].name);
 447                         return -EIO;
 448                 }
 449         }
 450
 451         /* Initialize GDC with default values */
 452
 453         for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
 454                 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
 455                 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
 456                 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
 457                 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
 458
 459                 writel(val0 | (val1 << 16),
 460                        base + IMGU_REG_GDC_LUT_BASE + i * 8);
 461                 writel(val2 | (val3 << 16),
 462                        base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
 463         }
 464
 465         return 0;
 466 }
 467
 468 /* Boot the given IPU3 CSS SP */
 469 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
 470 {
 471         void __iomem *const base = css->base;
 472         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
 473         struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
 474                 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
 475                         + bi->blob.data_source,
 476                 .dmem_data_addr = bi->blob.data_target,
 477                 .dmem_bss_addr = bi->blob.bss_target,
 478                 .data_size = bi->blob.data_size,
 479                 .bss_size = bi->blob.bss_size,
 480                 .sp_id = sp,
 481         };
 482
 483         writes(&dmem_cfg, sizeof(dmem_cfg), base +
 484                IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
 485
 486         writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
 487
 488         writel(readl(base + IMGU_REG_SP_CTRL(sp))
 489                 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
 490
 491         if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
 492                          + bi->info.sp.sw_state,
 493                          ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
 494                 return -EIO;
 495
 496         return 0;
 497 }
 498
 499 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
 500 static int imgu_css_hw_start(struct imgu_css *css)
 501 {
 502         static const u32 event_mask =
 503                 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
 504                 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
 505                 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
 506                 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
 507                 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
 508                 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
 509                 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
 510                 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
 511                 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
 512                 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
 513                 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
 514                 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
 515
 516         void __iomem *const base = css->base;
 517         struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
 518         unsigned int i;
 519
 520         writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
 521
 522         /* Start bootloader */
 523
 524         writel(IMGU_ABI_BL_SWSTATE_BUSY,
 525                base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
 526         writel(IMGU_NUM_SP,
 527                base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
 528
 529         for (i = 0; i < IMGU_NUM_SP; i++) {
 530                 int j = IMGU_NUM_SP - i - 1;    /* load sp1 first, then sp0 */
 531                 struct imgu_fw_info *sp =
 532                                         &css->fwp->binary_header[css->fw_sp[j]];
 533                 struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
 534                         .src_addr = css->binary[css->fw_sp[j]].daddr
 535                                 + sp->blob.text_source,
 536                         .size = sp->blob.text_size,
 537                         .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
 538                         .dst_addr = IMGU_SP_PMEM_BASE(j),
 539                 };
 540
 541                 writes(&dma_cmd, sizeof(dma_cmd),
 542                        base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
 543                        bl->info.bl.dma_cmd_list);
 544         }
 545
 546         writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
 547
 548         writel(readl(base + IMGU_REG_ISP_CTRL)
 549                 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
 550         if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
 551                          + bl->info.bl.sw_state, ~0,
 552                          IMGU_ABI_BL_SWSTATE_OK)) {
 553                 dev_err(css->dev, "failed to start bootloader\n");
 554                 return -EIO;
 555         }
 556
 557         /* Start ISP */
 558
 559         memset(css->xmem_sp_group_ptrs.vaddr, 0,
 560                sizeof(struct imgu_abi_sp_group));
 561
 562         bi = &css->fwp->binary_header[css->fw_sp[0]];
 563
 564         writel(css->xmem_sp_group_ptrs.daddr,
 565                base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
 566
 567         writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
 568                base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
 569         writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
 570
 571         if (imgu_css_hw_start_sp(css, 0))
 572                 return -EIO;
 573
 574         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
 575         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
 576                 bi->info.sp.host_sp_queues_initialized);
 577         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
 578         writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
 579         writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
 580                 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 581
 582         /* Enable all events for all queues */
 583
 584         for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
 585                 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
 586                         + bi->info.sp.host_sp_com
 587                         + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
 588         writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
 589                 bi->info.sp.host_sp_queues_initialized);
 590
 591         /* Start SP1 */
 592
 593         bi = &css->fwp->binary_header[css->fw_sp[1]];
 594
 595         writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
 596                base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
 597
 598         if (imgu_css_hw_start_sp(css, 1))
 599                 return -EIO;
 600
 601         writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
 602                 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 603
 604         return 0;
 605 }
 606
 607 static void imgu_css_hw_stop(struct imgu_css *css)
 608 {
 609         void __iomem *const base = css->base;
 610         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
 611
 612         /* Stop fw */
 613         writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
 614                base + IMGU_REG_SP_DMEM_BASE(0) +
 615                bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 616         if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
 617                          IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
 618                 dev_err(css->dev, "wait sp0 idle timeout.\n");
 619         if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
 620                   IMGU_ABI_SP_SWSTATE_TERMINATED)
 621                 dev_err(css->dev, "sp0 is not terminated.\n");
 622         if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
 623                          IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
 624                 dev_err(css->dev, "wait isp idle timeout\n");
 625 }
 626
 627 static void imgu_css_hw_cleanup(struct imgu_css *css)
 628 {
 629         void __iomem *const base = css->base;
 630
 631         /** Reset CSS **/
 632
 633         /* Clear the CSS busy signal */
 634         readl(base + IMGU_REG_GP_BUSY);
 635         writel(0, base + IMGU_REG_GP_BUSY);
 636
 637         /* Wait for idle signal */
 638         if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 639                          IMGU_STATE_IDLE_STS))
 640                 dev_err(css->dev, "failed to shut down hw cleanly\n");
 641
 642         /* Reset the css */
 643         writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
 644                base + IMGU_REG_PM_CTRL);
 645
 646         usleep_range(200, 300);
 647 }
 648
 649 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
 650 {
 651         struct imgu_device *imgu = dev_get_drvdata(css->dev);
 652         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
 653         unsigned int i;
 654
 655         imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
 656         imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
 657         imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
 658         imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
 659
 660         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
 661                 imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
 662 }
 663
 664 /*
 665  * This function initializes various stages of the
 666  * IPU3 CSS ISP pipeline
 667  */
 668 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
 669 {
 670         static const int BYPC = 2;      /* Bytes per component */
 671         static const struct imgu_abi_buffer_sp buffer_sp_init = {
 672                 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
 673                 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
 674         };
 675
 676         struct imgu_abi_isp_iterator_config *cfg_iter;
 677         struct imgu_abi_isp_ref_config *cfg_ref;
 678         struct imgu_abi_isp_dvs_config *cfg_dvs;
 679         struct imgu_abi_isp_tnr3_config *cfg_tnr;
 680         struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
 681         struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
 682
 683         const int stage = 0;
 684         unsigned int i, j;
 685
 686         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
 687         struct imgu_css_queue *css_queue_in =
 688                         &css_pipe->queue[IPU3_CSS_QUEUE_IN];
 689         struct imgu_css_queue *css_queue_out =
 690                         &css_pipe->queue[IPU3_CSS_QUEUE_OUT];
 691         struct imgu_css_queue *css_queue_vf =
 692                         &css_pipe->queue[IPU3_CSS_QUEUE_VF];
 693         const struct imgu_fw_info *bi =
 694                         &css->fwp->binary_header[css_pipe->bindex];
 695         const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
 696
 697         struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
 698                 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
 699         struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
 700                 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
 701
 702         struct imgu_abi_isp_stage *isp_stage;
 703         struct imgu_abi_sp_stage *sp_stage;
 704         struct imgu_abi_sp_group *sp_group;
 705         struct imgu_abi_frames_sp *frames_sp;
 706         struct imgu_abi_frame_sp *frame_sp;
 707         struct imgu_abi_frame_sp_info *frame_sp_info;
 708
 709         const unsigned int bds_width_pad =
 710                                 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
 711                                       2 * IPU3_UAPI_ISP_VEC_ELEMS);
 712
 713         const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
 714         enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
 715         void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
 716
 717         struct imgu_device *imgu = dev_get_drvdata(css->dev);
 718
 719         dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
 720
 721         /* Configure iterator */
 722
 723         cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 724                                                &cofs->dmem.iterator,
 725                                                sizeof(*cfg_iter), vaddr);
 726         if (!cfg_iter)
 727                 goto bad_firmware;
 728
 729         frame_sp_info = &cfg_iter->input_info;
 730         frame_sp_info->res.width        = css_queue_in->fmt.mpix.width;
 731         frame_sp_info->res.height       = css_queue_in->fmt.mpix.height;
 732         frame_sp_info->padded_width     = css_queue_in->width_pad;
 733         frame_sp_info->format           = css_queue_in->css_fmt->frame_format;
 734         frame_sp_info->raw_bit_depth    = css_queue_in->css_fmt->bit_depth;
 735         frame_sp_info->raw_bayer_order  = css_queue_in->css_fmt->bayer_order;
 736         frame_sp_info->raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 737
 738         frame_sp_info = &cfg_iter->internal_info;
 739         frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
 740         frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
 741         frame_sp_info->padded_width     = bds_width_pad;
 742         frame_sp_info->format           = css_queue_out->css_fmt->frame_format;
 743         frame_sp_info->raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
 744         frame_sp_info->raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
 745         frame_sp_info->raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 746
 747         frame_sp_info = &cfg_iter->output_info;
 748         frame_sp_info->res.width        = css_queue_out->fmt.mpix.width;
 749         frame_sp_info->res.height       = css_queue_out->fmt.mpix.height;
 750         frame_sp_info->padded_width     = css_queue_out->width_pad;
 751         frame_sp_info->format           = css_queue_out->css_fmt->frame_format;
 752         frame_sp_info->raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
 753         frame_sp_info->raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
 754         frame_sp_info->raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 755
 756         frame_sp_info = &cfg_iter->vf_info;
 757         frame_sp_info->res.width        = css_queue_vf->fmt.mpix.width;
 758         frame_sp_info->res.height       = css_queue_vf->fmt.mpix.height;
 759         frame_sp_info->padded_width     = css_queue_vf->width_pad;
 760         frame_sp_info->format           = css_queue_vf->css_fmt->frame_format;
 761         frame_sp_info->raw_bit_depth    = css_queue_vf->css_fmt->bit_depth;
 762         frame_sp_info->raw_bayer_order  = css_queue_vf->css_fmt->bayer_order;
 763         frame_sp_info->raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 764
 765         cfg_iter->dvs_envelope.width =
 766                                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
 767         cfg_iter->dvs_envelope.height =
 768                                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
 769
 770         /* Configure reference (delay) frames */
 771
 772         cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 773                                               &cofs->dmem.ref,
 774                                               sizeof(*cfg_ref), vaddr);
 775         if (!cfg_ref)
 776                 goto bad_firmware;
 777
 778         cfg_ref->port_b.crop = 0;
 779         cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
 780         cfg_ref->port_b.width =
 781                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
 782         cfg_ref->port_b.stride =
 783                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
 784         cfg_ref->width_a_over_b =
 785                                 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
 786         cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
 787         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
 788                 cfg_ref->ref_frame_addr_y[i] =
 789                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
 790                 cfg_ref->ref_frame_addr_c[i] =
 791                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
 792                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
 793                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
 794         }
 795         for (; i < IMGU_ABI_FRAMES_REF; i++) {
 796                 cfg_ref->ref_frame_addr_y[i] = 0;
 797                 cfg_ref->ref_frame_addr_c[i] = 0;
 798         }
 799
 800         /* Configure DVS (digital video stabilization) */
 801
 802         cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 803                                               &cofs->dmem.dvs, sizeof(*cfg_dvs),
 804                                               vaddr);
 805         if (!cfg_dvs)
 806                 goto bad_firmware;
 807
 808         cfg_dvs->num_horizontal_blocks =
 809                         ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
 810                                            IMGU_DVS_BLOCK_W), 2);
 811         cfg_dvs->num_vertical_blocks =
 812                         DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
 813                                      IMGU_DVS_BLOCK_H);
 814
 815         /* Configure TNR (temporal noise reduction) */
 816
 817         if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
 818                 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 819                                                       &cofs->dmem.tnr3,
 820                                                       sizeof(*cfg_tnr),
 821                                                       vaddr);
 822                 if (!cfg_tnr)
 823                         goto bad_firmware;
 824
 825                 cfg_tnr->port_b.crop = 0;
 826                 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
 827                 cfg_tnr->port_b.width =
 828                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
 829                 cfg_tnr->port_b.stride =
 830                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
 831                 cfg_tnr->width_a_over_b =
 832                         IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
 833                 cfg_tnr->frame_height =
 834                         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
 835                 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
 836                 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
 837                         cfg_tnr->frame_addr[i] =
 838                                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
 839                                         .mem[i].daddr;
 840                 for (; i < IMGU_ABI_FRAMES_TNR; i++)
 841                         cfg_tnr->frame_addr[i] = 0;
 842         }
 843
 844         /* Configure ref dmem state parameters */
 845
 846         cfg = IMGU_ABI_PARAM_CLASS_STATE;
 847         vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
 848
 849         cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 850                                                     &sofs->dmem.ref,
 851                                                     sizeof(*cfg_ref_state),
 852                                                     vaddr);
 853         if (!cfg_ref_state)
 854                 goto bad_firmware;
 855
 856         cfg_ref_state->ref_in_buf_idx = 0;
 857         cfg_ref_state->ref_out_buf_idx = 1;
 858
 859         /* Configure tnr dmem state parameters */
 860         if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
 861                 cfg_tnr_state =
 862                         imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 863                                                     &sofs->dmem.tnr3,
 864                                                     sizeof(*cfg_tnr_state),
 865                                                     vaddr);
 866                 if (!cfg_tnr_state)
 867                         goto bad_firmware;
 868
 869                 cfg_tnr_state->in_bufidx = 0;
 870                 cfg_tnr_state->out_bufidx = 1;
 871                 cfg_tnr_state->bypass_filter = 0;
 872                 cfg_tnr_state->total_frame_counter = 0;
 873                 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
 874                         cfg_tnr_state->buffer_frame_counter[i] = 0;
 875         }
 876
 877         /* Configure ISP stage */
 878
 879         isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
 880         memset(isp_stage, 0, sizeof(*isp_stage));
 881         isp_stage->blob_info = bi->blob;
 882         isp_stage->binary_info = bi->info.isp.sp;
 883         strscpy(isp_stage->binary_name,
 884                 (char *)css->fwp + bi->blob.prog_name_offset,
 885                 sizeof(isp_stage->binary_name));
 886         isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
 887         for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
 888                 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
 889                         isp_stage->mem_initializers.params[i][j].address =
 890                                         css_pipe->binary_params_cs[i - 1][j].daddr;
 891
 892         /* Configure SP stage */
 893
 894         sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
 895         memset(sp_stage, 0, sizeof(*sp_stage));
 896
 897         frames_sp = &sp_stage->frames;
 898         frames_sp->in.buf_attr = buffer_sp_init;
 899         for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
 900                 frames_sp->out[i].buf_attr = buffer_sp_init;
 901         frames_sp->out_vf.buf_attr = buffer_sp_init;
 902         frames_sp->s3a_buf = buffer_sp_init;
 903         frames_sp->dvs_buf = buffer_sp_init;
 904
 905         sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
 906         sp_stage->num = stage;
 907         sp_stage->isp_online = 0;
 908         sp_stage->isp_copy_vf = 0;
 909         sp_stage->isp_copy_output = 0;
 910
 911         sp_stage->enable.vf_output = css_pipe->vf_output_en;
 912
 913         frames_sp->effective_in_res.width =
 914                                 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
 915         frames_sp->effective_in_res.height =
 916                                 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
 917
 918         frame_sp = &frames_sp->in;
 919         frame_sp->info.res.width        = css_queue_in->fmt.mpix.width;
 920         frame_sp->info.res.height       = css_queue_in->fmt.mpix.height;
 921         frame_sp->info.padded_width     = css_queue_in->width_pad;
 922         frame_sp->info.format           = css_queue_in->css_fmt->frame_format;
 923         frame_sp->info.raw_bit_depth    = css_queue_in->css_fmt->bit_depth;
 924         frame_sp->info.raw_bayer_order  = css_queue_in->css_fmt->bayer_order;
 925         frame_sp->info.raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 926         frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
 927         frame_sp->buf_attr.buf_type     = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
 928
 929         frame_sp = &frames_sp->out[0];
 930         frame_sp->info.res.width        = css_queue_out->fmt.mpix.width;
 931         frame_sp->info.res.height       = css_queue_out->fmt.mpix.height;
 932         frame_sp->info.padded_width     = css_queue_out->width_pad;
 933         frame_sp->info.format           = css_queue_out->css_fmt->frame_format;
 934         frame_sp->info.raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
 935         frame_sp->info.raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
 936         frame_sp->info.raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 937         frame_sp->planes.nv.uv.offset   = css_queue_out->width_pad *
 938                                           css_queue_out->fmt.mpix.height;
 939         frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
 940         frame_sp->buf_attr.buf_type     = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
 941
 942         frame_sp = &frames_sp->out[1];
 943         frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
 944
 945         frame_sp_info = &frames_sp->internal_frame_info;
 946         frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
 947         frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
 948         frame_sp_info->padded_width     = bds_width_pad;
 949         frame_sp_info->format           = css_queue_out->css_fmt->frame_format;
 950         frame_sp_info->raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
 951         frame_sp_info->raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
 952         frame_sp_info->raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 953
 954         frame_sp = &frames_sp->out_vf;
 955         frame_sp->info.res.width        = css_queue_vf->fmt.mpix.width;
 956         frame_sp->info.res.height       = css_queue_vf->fmt.mpix.height;
 957         frame_sp->info.padded_width     = css_queue_vf->width_pad;
 958         frame_sp->info.format           = css_queue_vf->css_fmt->frame_format;
 959         frame_sp->info.raw_bit_depth    = css_queue_vf->css_fmt->bit_depth;
 960         frame_sp->info.raw_bayer_order  = css_queue_vf->css_fmt->bayer_order;
 961         frame_sp->info.raw_type         = IMGU_ABI_RAW_TYPE_BAYER;
 962         frame_sp->planes.yuv.u.offset   = css_queue_vf->width_pad *
 963                                           css_queue_vf->fmt.mpix.height;
 964         frame_sp->planes.yuv.v.offset   = css_queue_vf->width_pad *
 965                                           css_queue_vf->fmt.mpix.height * 5 / 4;
 966         frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
 967         frame_sp->buf_attr.buf_type     = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
 968
 969         frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
 970         frames_sp->s3a_buf.buf_type     = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
 971
 972         frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
 973         frames_sp->dvs_buf.buf_type     = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
 974
 975         sp_stage->dvs_envelope.width =
 976                                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
 977         sp_stage->dvs_envelope.height =
 978                                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
 979
 980         sp_stage->isp_pipe_version =
 981                                 bi->info.isp.sp.pipeline.isp_pipe_version;
 982         sp_stage->isp_deci_log_factor =
 983                         clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
 984                                       IMGU_MAX_BQ_GRID_WIDTH),
 985                                   fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
 986                                       IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
 987         sp_stage->isp_vf_downscale_bits = 0;
 988         sp_stage->if_config_index = 255;
 989         sp_stage->sp_enable_xnr = 0;
 990         sp_stage->num_stripes = stripes;
 991         sp_stage->enable.s3a = 1;
 992         sp_stage->enable.dvs_stats = 0;
 993
 994         sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
 995         sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
 996         sp_stage->isp_stage_addr =
 997                 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
 998
 999         /* Configure SP group */
1000
1001         sp_group = css->xmem_sp_group_ptrs.vaddr;
1002         memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1003
1004         sp_group->pipe[pipe].num_stages = 1;
1005         sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1006         sp_group->pipe[pipe].thread_id = pipe;
1007         sp_group->pipe[pipe].pipe_num = pipe;
1008         sp_group->pipe[pipe].num_execs = -1;
1009         sp_group->pipe[pipe].pipe_qos_config = -1;
1010         sp_group->pipe[pipe].required_bds_factor = 0;
1011         sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1012         sp_group->pipe[pipe].inout_port_config =
1013                                         IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1014                                         IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1015         sp_group->pipe[pipe].scaler_pp_lut = 0;
1016         sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1017         sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1018         sp_group->pipe[pipe].sp_stage_addr[stage] =
1019                         css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1020         sp_group->pipe[pipe].pipe_config =
1021                         bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1022         sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1023
1024         /* Initialize parameter pools */
1025
1026         if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1027                                sizeof(struct imgu_abi_parameter_set_info)) ||
1028             imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1029                                sizeof(struct imgu_abi_acc_param)) ||
1030             imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1031                                sizeof(struct imgu_abi_gdc_warp_param) *
1032                                3 * cfg_dvs->num_horizontal_blocks / 2 *
1033                                cfg_dvs->num_vertical_blocks) ||
1034             imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1035                                imgu_css_fw_obgrid_size(
1036                                &css->fwp->binary_header[css_pipe->bindex])))
1037                 goto out_of_memory;
1038
1039         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1040                 if (imgu_css_pool_init(imgu,
1041                                        &css_pipe->pool.binary_params_p[i],
1042                                        bi->info.isp.sp.mem_initializers.params
1043                                        [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1044                         goto out_of_memory;
1045
1046         return 0;
1047
1048 bad_firmware:
1049         imgu_css_pipeline_cleanup(css, pipe);
1050         return -EPROTO;
1051
1052 out_of_memory:
1053         imgu_css_pipeline_cleanup(css, pipe);
1054         return -ENOMEM;
1055 }
1056
1057 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1058 {
1059         static const unsigned int sp;
1060         void __iomem *const base = css->base;
1061         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1062         struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1063                                                 bi->info.sp.host_sp_queue;
1064
1065         return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1066                             readb(&q->host2sp_evtq_info.end);
1067 }
1068
1069 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
1070 static int imgu_css_queue_data(struct imgu_css *css,
1071                                int queue, int thread, u32 data)
1072 {
1073         static const unsigned int sp;
1074         void __iomem *const base = css->base;
1075         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1076         struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1077                                                 bi->info.sp.host_sp_queue;
1078         u8 size, start, end, end2;
1079
1080         if (queue >= 0) {
1081                 size = readb(&q->host2sp_bufq_info[thread][queue].size);
1082                 start = readb(&q->host2sp_bufq_info[thread][queue].start);
1083                 end = readb(&q->host2sp_bufq_info[thread][queue].end);
1084         } else {
1085                 size = readb(&q->host2sp_evtq_info.size);
1086                 start = readb(&q->host2sp_evtq_info.start);
1087                 end = readb(&q->host2sp_evtq_info.end);
1088         }
1089
1090         if (size == 0)
1091                 return -EIO;
1092
1093         end2 = (end + 1) % size;
1094         if (end2 == start)
1095                 return -EBUSY;  /* Queue full */
1096
1097         if (queue >= 0) {
1098                 writel(data, &q->host2sp_bufq[thread][queue][end]);
1099                 writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1100         } else {
1101                 writel(data, &q->host2sp_evtq[end]);
1102                 writeb(end2, &q->host2sp_evtq_info.end);
1103         }
1104
1105         return 0;
1106 }
1107
1108 /* Receive data using given buffer queue, or if queue < 0, event queue. */
1109 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1110 {
1111         static const unsigned int sp;
1112         void __iomem *const base = css->base;
1113         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1114         struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1115                                                 bi->info.sp.host_sp_queue;
1116         u8 size, start, end, start2;
1117
1118         if (queue >= 0) {
1119                 size = readb(&q->sp2host_bufq_info[queue].size);
1120                 start = readb(&q->sp2host_bufq_info[queue].start);
1121                 end = readb(&q->sp2host_bufq_info[queue].end);
1122         } else {
1123                 size = readb(&q->sp2host_evtq_info.size);
1124                 start = readb(&q->sp2host_evtq_info.start);
1125                 end = readb(&q->sp2host_evtq_info.end);
1126         }
1127
1128         if (size == 0)
1129                 return -EIO;
1130
1131         if (end == start)
1132                 return -EBUSY;  /* Queue empty */
1133
1134         start2 = (start + 1) % size;
1135
1136         if (queue >= 0) {
1137                 *data = readl(&q->sp2host_bufq[queue][start]);
1138                 writeb(start2, &q->sp2host_bufq_info[queue].start);
1139         } else {
1140                 int r;
1141
1142                 *data = readl(&q->sp2host_evtq[start]);
1143                 writeb(start2, &q->sp2host_evtq_info.start);
1144
1145                 /* Acknowledge events dequeued from event queue */
1146                 r = imgu_css_queue_data(css, queue, 0,
1147                                         IMGU_ABI_EVENT_EVENT_DEQUEUED);
1148                 if (r < 0)
1149                         return r;
1150         }
1151
1152         return 0;
1153 }
1154
1155 /* Free binary-specific resources */
1156 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1157 {
1158         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1159         unsigned int i, j;
1160
1161         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1162
1163         for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1164                 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1165                         imgu_dmamap_free(imgu,
1166                                          &css_pipe->binary_params_cs[j][i]);
1167
1168         j = IPU3_CSS_AUX_FRAME_REF;
1169         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1170                 imgu_dmamap_free(imgu,
1171                                  &css_pipe->aux_frames[j].mem[i]);
1172
1173         j = IPU3_CSS_AUX_FRAME_TNR;
1174         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1175                 imgu_dmamap_free(imgu,
1176                                  &css_pipe->aux_frames[j].mem[i]);
1177 }
1178
1179 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1180 {
1181         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1182         unsigned int i, j;
1183
1184         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1185
1186         for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1187              j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1188                 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1189                         if (!imgu_dmamap_alloc(imgu,
1190                                                &css_pipe->binary_params_cs[j - 1][i],
1191                                                CSS_ABI_SIZE))
1192                                 goto out_of_memory;
1193
1194         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1195                 if (!imgu_dmamap_alloc(imgu,
1196                                        &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1197                                        CSS_BDS_SIZE))
1198                         goto out_of_memory;
1199
1200         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1201                 if (!imgu_dmamap_alloc(imgu,
1202                                        &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1203                                        CSS_GDC_SIZE))
1204                         goto out_of_memory;
1205
1206         return 0;
1207
1208 out_of_memory:
1209         imgu_css_binary_cleanup(css, pipe);
1210         return -ENOMEM;
1211 }
1212
1213 /* allocate binary-specific resources */
1214 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1215 {
1216         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1217         struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1218         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1219         int i, j, size;
1220         static const int BYPC = 2;      /* Bytes per component */
1221         unsigned int w, h;
1222
1223         /* Allocate parameter memory blocks for this binary */
1224
1225         for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1226                 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1227                         if (imgu_css_dma_buffer_resize(
1228                             imgu,
1229                             &css_pipe->binary_params_cs[j - 1][i],
1230                             bi->info.isp.sp.mem_initializers.params[j][i].size))
1231                                 goto out_of_memory;
1232                 }
1233
1234         /* Allocate internal frame buffers */
1235
1236         /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1237         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1238         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1239                                         css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1240         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1241                                 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1242                                       IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1243         h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1244         w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1245                   2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1246         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1247                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1248         size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1249         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1250                 if (imgu_css_dma_buffer_resize(
1251                         imgu,
1252                         &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1253                         size))
1254                         goto out_of_memory;
1255
1256         /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1257         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1258         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1259                         roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1260                                 bi->info.isp.sp.block.block_width *
1261                                 IPU3_UAPI_ISP_VEC_ELEMS);
1262         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1263                         roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1264                                 bi->info.isp.sp.block.output_block_height);
1265
1266         w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1267         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1268         h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1269         size = w * ALIGN(h * 3 / 2 + 3, 2);     /* +3 for vf_pp prefetch */
1270         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1271                 if (imgu_css_dma_buffer_resize(
1272                         imgu,
1273                         &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1274                         size))
1275                         goto out_of_memory;
1276
1277         return 0;
1278
1279 out_of_memory:
1280         imgu_css_binary_cleanup(css, pipe);
1281         return -ENOMEM;
1282 }
1283
1284 int imgu_css_start_streaming(struct imgu_css *css)
1285 {
1286         u32 data;
1287         int r, pipe;
1288
1289         if (css->streaming)
1290                 return -EPROTO;
1291
1292         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1293                 r = imgu_css_binary_setup(css, pipe);
1294                 if (r < 0)
1295                         return r;
1296         }
1297
1298         r = imgu_css_hw_init(css);
1299         if (r < 0)
1300                 return r;
1301
1302         r = imgu_css_hw_start(css);
1303         if (r < 0)
1304                 goto fail;
1305
1306         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1307                 r = imgu_css_pipeline_init(css, pipe);
1308                 if (r < 0)
1309                         goto fail;
1310         }
1311
1312         css->streaming = true;
1313
1314         imgu_css_hw_enable_irq(css);
1315
1316         /* Initialize parameters to default */
1317         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1318                 r = imgu_css_set_parameters(css, pipe, NULL);
1319                 if (r < 0)
1320                         goto fail;
1321         }
1322
1323         while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1324                 ;
1325         if (r != -EBUSY)
1326                 goto fail;
1327
1328         while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1329                 ;
1330         if (r != -EBUSY)
1331                 goto fail;
1332
1333         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1334                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1335                                         IMGU_ABI_EVENT_START_STREAM |
1336                                         pipe << 16);
1337                 if (r < 0)
1338                         goto fail;
1339         }
1340
1341         return 0;
1342
1343 fail:
1344         css->streaming = false;
1345         imgu_css_hw_cleanup(css);
1346         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1347                 imgu_css_pipeline_cleanup(css, pipe);
1348                 imgu_css_binary_cleanup(css, pipe);
1349         }
1350
1351         return r;
1352 }
1353
1354 void imgu_css_stop_streaming(struct imgu_css *css)
1355 {
1356         struct imgu_css_buffer *b, *b0;
1357         int q, r, pipe;
1358
1359         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1360                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1361                                         IMGU_ABI_EVENT_STOP_STREAM);
1362                 if (r < 0)
1363                         dev_warn(css->dev, "failed on stop stream event\n");
1364         }
1365
1366         if (!css->streaming)
1367                 return;
1368
1369         imgu_css_hw_stop(css);
1370
1371         imgu_css_hw_cleanup(css);
1372
1373         for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1374                 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1375
1376                 imgu_css_pipeline_cleanup(css, pipe);
1377
1378                 spin_lock(&css_pipe->qlock);
1379                 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1380                         list_for_each_entry_safe(b, b0,
1381                                                  &css_pipe->queue[q].bufs,
1382                                                  list) {
1383                                 b->state = IPU3_CSS_BUFFER_FAILED;
1384                                 list_del(&b->list);
1385                         }
1386                 spin_unlock(&css_pipe->qlock);
1387         }
1388
1389         css->streaming = false;
1390 }
1391
1392 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1393 {
1394         int q;
1395         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1396
1397         spin_lock(&css_pipe->qlock);
1398         for (q = 0; q < IPU3_CSS_QUEUES; q++)
1399                 if (!list_empty(&css_pipe->queue[q].bufs))
1400                         break;
1401         spin_unlock(&css_pipe->qlock);
1402         return (q == IPU3_CSS_QUEUES);
1403 }
1404
1405 bool imgu_css_queue_empty(struct imgu_css *css)
1406 {
1407         unsigned int pipe;
1408         bool ret = false;
1409
1410         for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1411                 ret &= imgu_css_pipe_queue_empty(css, pipe);
1412
1413         return ret;
1414 }
1415
1416 bool imgu_css_is_streaming(struct imgu_css *css)
1417 {
1418         return css->streaming;
1419 }
1420
1421 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1422 {
1423         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1424         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1425         unsigned int p, q, i;
1426
1427         /* Allocate and map common structures with imgu hardware */
1428         for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1429                 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1430                         if (!imgu_dmamap_alloc(imgu,
1431                                                &css_pipe->xmem_sp_stage_ptrs[p][i],
1432                                                sizeof(struct imgu_abi_sp_stage)))
1433                                 return -ENOMEM;
1434                         if (!imgu_dmamap_alloc(imgu,
1435                                                &css_pipe->xmem_isp_stage_ptrs[p][i],
1436                                                sizeof(struct imgu_abi_isp_stage)))
1437                                 return -ENOMEM;
1438                 }
1439
1440         if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1441                                ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1442                                      IMGU_ABI_ISP_DDR_WORD_BYTES)))
1443                 return -ENOMEM;
1444
1445         for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1446                 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1447
1448                 for (i = 0; i < abi_buf_num; i++)
1449                         if (!imgu_dmamap_alloc(imgu,
1450                                                &css_pipe->abi_buffers[q][i],
1451                                                sizeof(struct imgu_abi_buffer)))
1452                                 return -ENOMEM;
1453         }
1454
1455         if (imgu_css_binary_preallocate(css, pipe)) {
1456                 imgu_css_binary_cleanup(css, pipe);
1457                 return -ENOMEM;
1458         }
1459
1460         return 0;
1461 }
1462
1463 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1464 {
1465         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1466         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1467         unsigned int p, q, i, abi_buf_num;
1468
1469         imgu_css_binary_cleanup(css, pipe);
1470
1471         for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1472                 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1473                 for (i = 0; i < abi_buf_num; i++)
1474                         imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1475         }
1476
1477         for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1478                 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1479                         imgu_dmamap_free(imgu,
1480                                          &css_pipe->xmem_sp_stage_ptrs[p][i]);
1481                         imgu_dmamap_free(imgu,
1482                                          &css_pipe->xmem_isp_stage_ptrs[p][i]);
1483                 }
1484
1485         imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1486 }
1487
1488 void imgu_css_cleanup(struct imgu_css *css)
1489 {
1490         struct imgu_device *imgu = dev_get_drvdata(css->dev);
1491         unsigned int pipe;
1492
1493         imgu_css_stop_streaming(css);
1494         for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1495                 imgu_css_pipe_cleanup(css, pipe);
1496         imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1497         imgu_css_fw_cleanup(css);
1498 }
1499
1500 int imgu_css_init(struct device *dev, struct imgu_css *css,
1501                   void __iomem *base, int length)
1502 {
1503         struct imgu_device *imgu = dev_get_drvdata(dev);
1504         int r, q, pipe;
1505
1506         /* Initialize main data structure */
1507         css->dev = dev;
1508         css->base = base;
1509         css->iomem_length = length;
1510
1511         for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1512                 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1513
1514                 css_pipe->vf_output_en = false;
1515                 spin_lock_init(&css_pipe->qlock);
1516                 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1517                 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1518                 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1519                         r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1520                         if (r)
1521                                 return r;
1522                 }
1523                 r = imgu_css_map_init(css, pipe);
1524                 if (r) {
1525                         imgu_css_cleanup(css);
1526                         return r;
1527                 }
1528         }
1529         if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1530                                sizeof(struct imgu_abi_sp_group)))
1531                 return -ENOMEM;
1532
1533         r = imgu_css_fw_init(css);
1534         if (r)
1535                 return r;
1536
1537         return 0;
1538 }
1539
1540 static u32 imgu_css_adjust(u32 res, u32 align)
1541 {
1542         u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1543
1544         return DIV_ROUND_CLOSEST(val, align) * align;
1545 }
1546
1547 /* Select a binary matching the required resolutions and formats */
1548 static int imgu_css_find_binary(struct imgu_css *css,
1549                                 unsigned int pipe,
1550                                 struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1551                                 struct v4l2_rect rects[IPU3_CSS_RECTS])
1552 {
1553         const int binary_nr = css->fwp->file_header.binary_nr;
1554         unsigned int binary_mode =
1555                 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1556                 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1557         const struct v4l2_pix_format_mplane *in =
1558                                         &queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1559         const struct v4l2_pix_format_mplane *out =
1560                                         &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1561         const struct v4l2_pix_format_mplane *vf =
1562                                         &queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1563         u32 stripe_w = 0, stripe_h = 0;
1564         const char *name;
1565         int i, j;
1566
1567         if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1568                 return -EINVAL;
1569
1570         /* Find out the strip size boundary */
1571         for (i = 0; i < binary_nr; i++) {
1572                 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1573
1574                 u32 max_width = bi->info.isp.sp.output.max_width;
1575                 u32 max_height = bi->info.isp.sp.output.max_height;
1576
1577                 if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1578                         stripe_w = stripe_w ?
1579                                 min(stripe_w, max_width) : max_width;
1580                         stripe_h = stripe_h ?
1581                                 min(stripe_h, max_height) : max_height;
1582                 }
1583         }
1584
1585         for (i = 0; i < binary_nr; i++) {
1586                 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1587                 enum imgu_abi_frame_format q_fmt;
1588
1589                 name = (void *)css->fwp + bi->blob.prog_name_offset;
1590
1591                 /* Check that binary supports memory-to-memory processing */
1592                 if (bi->info.isp.sp.input.source !=
1593                     IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1594                         continue;
1595
1596                 /* Check that binary supports raw10 input */
1597                 if (!bi->info.isp.sp.enable.input_feeder &&
1598                     !bi->info.isp.sp.enable.input_raw)
1599                         continue;
1600
1601                 /* Check binary mode */
1602                 if (bi->info.isp.sp.pipeline.mode != binary_mode)
1603                         continue;
1604
1605                 /* Since input is RGGB bayer, need to process colors */
1606                 if (bi->info.isp.sp.enable.luma_only)
1607                         continue;
1608
1609                 if (in->width < bi->info.isp.sp.input.min_width ||
1610                     in->width > bi->info.isp.sp.input.max_width ||
1611                     in->height < bi->info.isp.sp.input.min_height ||
1612                     in->height > bi->info.isp.sp.input.max_height)
1613                         continue;
1614
1615                 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1616                         if (bi->info.isp.num_output_pins <= 0)
1617                                 continue;
1618
1619                         q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1620                         for (j = 0; j < bi->info.isp.num_output_formats; j++)
1621                                 if (bi->info.isp.output_formats[j] == q_fmt)
1622                                         break;
1623                         if (j >= bi->info.isp.num_output_formats)
1624                                 continue;
1625
1626                         if (out->width < bi->info.isp.sp.output.min_width ||
1627                             out->width > bi->info.isp.sp.output.max_width ||
1628                             out->height < bi->info.isp.sp.output.min_height ||
1629                             out->height > bi->info.isp.sp.output.max_height)
1630                                 continue;
1631
1632                         if (out->width > bi->info.isp.sp.internal.max_width ||
1633                             out->height > bi->info.isp.sp.internal.max_height)
1634                                 continue;
1635                 }
1636
1637                 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1638                         if (bi->info.isp.num_output_pins <= 1)
1639                                 continue;
1640
1641                         q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1642                         for (j = 0; j < bi->info.isp.num_output_formats; j++)
1643                                 if (bi->info.isp.output_formats[j] == q_fmt)
1644                                         break;
1645                         if (j >= bi->info.isp.num_output_formats)
1646                                 continue;
1647
1648                         if (vf->width < bi->info.isp.sp.output.min_width ||
1649                             vf->width > bi->info.isp.sp.output.max_width ||
1650                             vf->height < bi->info.isp.sp.output.min_height ||
1651                             vf->height > bi->info.isp.sp.output.max_height)
1652                                 continue;
1653                 }
1654
1655                 /* All checks passed, select the binary */
1656                 dev_dbg(css->dev, "using binary %s id = %u\n", name,
1657                         bi->info.isp.sp.id);
1658                 return i;
1659         }
1660
1661         /* Can not find suitable binary for these parameters */
1662         return -EINVAL;
1663 }
1664
1665 /*
1666  * Check that there is a binary matching requirements. Parameters may be
1667  * NULL indicating disabled input/output. Return negative if given
1668  * parameters can not be supported or on error, zero or positive indicating
1669  * found binary number. May modify the given parameters if not exact match
1670  * is found.
1671  */
1672 int imgu_css_fmt_try(struct imgu_css *css,
1673                      struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1674                      struct v4l2_rect *rects[IPU3_CSS_RECTS],
1675                      unsigned int pipe)
1676 {
1677         static const u32 EFF_ALIGN_W = 2;
1678         static const u32 BDS_ALIGN_W = 4;
1679         static const u32 OUT_ALIGN_W = 8;
1680         static const u32 OUT_ALIGN_H = 4;
1681         static const u32 VF_ALIGN_W  = 2;
1682         static const char *qnames[IPU3_CSS_QUEUES] = {
1683                 [IPU3_CSS_QUEUE_IN] = "in",
1684                 [IPU3_CSS_QUEUE_PARAMS]    = "params",
1685                 [IPU3_CSS_QUEUE_OUT] = "out",
1686                 [IPU3_CSS_QUEUE_VF] = "vf",
1687                 [IPU3_CSS_QUEUE_STAT_3A]   = "3a",
1688         };
1689         static const char *rnames[IPU3_CSS_RECTS] = {
1690                 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1691                 [IPU3_CSS_RECT_BDS]       = "bayer-domain scaled resolution",
1692                 [IPU3_CSS_RECT_ENVELOPE]  = "DVS envelope size",
1693                 [IPU3_CSS_RECT_GDC]  = "GDC output res",
1694         };
1695         struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1696         struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1697         struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1698         struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1699         struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1700         struct imgu_css_queue *q;
1701         struct v4l2_pix_format_mplane *in, *out, *vf;
1702         int i, s, ret;
1703
1704         q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
1705         if (!q)
1706                 return -ENOMEM;
1707
1708         in  = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1709         out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1710         vf  = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1711
1712         /* Adjust all formats, get statistics buffer sizes and formats */
1713         for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1714                 if (fmts[i])
1715                         dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1716                                 qnames[i], fmts[i]->width, fmts[i]->height,
1717                                 fmts[i]->pixelformat);
1718                 else
1719                         dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1720                                 qnames[i]);
1721                 if (imgu_css_queue_init(&q[i], fmts[i],
1722                                         IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1723                         dev_notice(css->dev, "can not initialize queue %s\n",
1724                                    qnames[i]);
1725                         ret = -EINVAL;
1726                         goto out;
1727                 }
1728         }
1729         for (i = 0; i < IPU3_CSS_RECTS; i++) {
1730                 if (rects[i]) {
1731                         dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1732                                 rnames[i], rects[i]->width, rects[i]->height);
1733                         r[i].width  = rects[i]->width;
1734                         r[i].height = rects[i]->height;
1735                 } else {
1736                         dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1737                                 rnames[i]);
1738                 }
1739                 /* For now, force known good resolutions */
1740                 r[i].left = 0;
1741                 r[i].top  = 0;
1742         }
1743
1744         /* Always require one input and vf only if out is also enabled */
1745         if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1746             !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1747                 dev_warn(css->dev, "required queues are disabled\n");
1748                 ret = -EINVAL;
1749                 goto out;
1750         }
1751
1752         if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1753                 out->width = in->width;
1754                 out->height = in->height;
1755         }
1756         if (eff->width <= 0 || eff->height <= 0) {
1757                 eff->width = in->width;
1758                 eff->height = in->height;
1759         }
1760         if (bds->width <= 0 || bds->height <= 0) {
1761                 bds->width = out->width;
1762                 bds->height = out->height;
1763         }
1764         if (gdc->width <= 0 || gdc->height <= 0) {
1765                 gdc->width = out->width;
1766                 gdc->height = out->height;
1767         }
1768
1769         in->width   = imgu_css_adjust(in->width, 1);
1770         in->height  = imgu_css_adjust(in->height, 1);
1771         eff->width  = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1772         eff->height = imgu_css_adjust(eff->height, 1);
1773         bds->width  = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1774         bds->height = imgu_css_adjust(bds->height, 1);
1775         gdc->width  = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1776         gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1777         out->width  = imgu_css_adjust(out->width, OUT_ALIGN_W);
1778         out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1779         vf->width   = imgu_css_adjust(vf->width, VF_ALIGN_W);
1780         vf->height  = imgu_css_adjust(vf->height, 1);
1781
1782         s = (bds->width - gdc->width) / 2;
1783         env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1784         s = (bds->height - gdc->height) / 2;
1785         env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1786
1787         ret = imgu_css_find_binary(css, pipe, q, r);
1788         if (ret < 0) {
1789                 dev_err(css->dev, "failed to find suitable binary\n");
1790                 ret = -EINVAL;
1791                 goto out;
1792         }
1793         css->pipes[pipe].bindex = ret;
1794
1795         dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1796                 css->pipes[pipe].bindex, pipe);
1797
1798         /* Final adjustment and set back the queried formats */
1799         for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1800                 if (fmts[i]) {
1801                         if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1802                                                 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1803                                 dev_err(css->dev,
1804                                         "final resolution adjustment failed\n");
1805                                 ret = -EINVAL;
1806                                 goto out;
1807                         }
1808                         *fmts[i] = q[i].fmt.mpix;
1809                 }
1810         }
1811
1812         for (i = 0; i < IPU3_CSS_RECTS; i++)
1813                 if (rects[i])
1814                         *rects[i] = r[i];
1815
1816         dev_dbg(css->dev,
1817                 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1818                  in->width, in->height, eff->width, eff->height,
1819                  bds->width, bds->height, gdc->width, gdc->height,
1820                  out->width, out->height, vf->width, vf->height);
1821
1822         ret = 0;
1823 out:
1824         kfree(q);
1825         return ret;
1826 }
1827
1828 int imgu_css_fmt_set(struct imgu_css *css,
1829                      struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1830                      struct v4l2_rect *rects[IPU3_CSS_RECTS],
1831                      unsigned int pipe)
1832 {
1833         struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1834         struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1835         int i, r;
1836         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1837
1838         for (i = 0; i < IPU3_CSS_RECTS; i++) {
1839                 if (rects[i])
1840                         rect_data[i] = *rects[i];
1841                 else
1842                         memset(&rect_data[i], 0, sizeof(rect_data[i]));
1843                 all_rects[i] = &rect_data[i];
1844         }
1845         r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1846         if (r < 0)
1847                 return r;
1848
1849         for (i = 0; i < IPU3_CSS_QUEUES; i++)
1850                 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1851                                         IPU3_CSS_QUEUE_TO_FLAGS(i)))
1852                         return -EINVAL;
1853         for (i = 0; i < IPU3_CSS_RECTS; i++) {
1854                 css_pipe->rect[i] = rect_data[i];
1855                 if (rects[i])
1856                         *rects[i] = rect_data[i];
1857         }
1858
1859         return 0;
1860 }
1861
1862 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1863 {
1864         switch (fmt->dataformat) {
1865         case V4L2_META_FMT_IPU3_PARAMS:
1866                 fmt->buffersize = sizeof(struct ipu3_uapi_params);
1867
1868                 /*
1869                  * Sanity check for the parameter struct size. This must
1870                  * not change!
1871                  */
1872                 BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1873
1874                 break;
1875         case V4L2_META_FMT_IPU3_STAT_3A:
1876                 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1877                 break;
1878         default:
1879                 return -EINVAL;
1880         }
1881
1882         return 0;
1883 }
1884
1885 /*
1886  * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1887  * called for the buffer. May be called from interrupt context.
1888  * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1889  * code on error conditions.
1890  */
1891 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1892                        struct imgu_css_buffer *b)
1893 {
1894         struct imgu_abi_buffer *abi_buf;
1895         struct imgu_addr_t *buf_addr;
1896         u32 data;
1897         int r;
1898         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1899
1900         if (!css->streaming)
1901                 return -EPROTO; /* CSS or buffer in wrong state */
1902
1903         if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1904                 return -EINVAL;
1905
1906         b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1907                                           pipe);
1908
1909         if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1910                 return -EIO;
1911         abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1912
1913         /* Fill struct abi_buffer for firmware */
1914         memset(abi_buf, 0, sizeof(*abi_buf));
1915
1916         buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1917         *(imgu_addr_t *)buf_addr = b->daddr;
1918
1919         if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1920                 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1921
1922         if (b->queue == IPU3_CSS_QUEUE_OUT)
1923                 abi_buf->payload.frame.padded_width =
1924                                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1925
1926         if (b->queue == IPU3_CSS_QUEUE_VF)
1927                 abi_buf->payload.frame.padded_width =
1928                                         css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1929
1930         spin_lock(&css_pipe->qlock);
1931         list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1932         spin_unlock(&css_pipe->qlock);
1933         b->state = IPU3_CSS_BUFFER_QUEUED;
1934
1935         data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1936         r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1937                                 pipe, data);
1938         if (r < 0)
1939                 goto queueing_failed;
1940
1941         data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1942                                               imgu_css_queues[b->queue].qid);
1943         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1944         if (r < 0)
1945                 goto queueing_failed;
1946
1947         dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1948                 b, b->queue, pipe);
1949
1950         return 0;
1951
1952 queueing_failed:
1953         b->state = (r == -EBUSY || r == -EAGAIN) ?
1954                 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1955         list_del(&b->list);
1956
1957         return r;
1958 }
1959
1960 /*
1961  * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1962  * should be called again, or -EBUSY which means that there are no more
1963  * buffers available. May be called from interrupt context.
1964  */
1965 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1966 {
1967         static const unsigned char evtype_to_queue[] = {
1968                 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1969                 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1970                 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1971                 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1972         };
1973         struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
1974         u32 event, daddr;
1975         int evtype, pipe, pipeid, queue, qid, r;
1976         struct imgu_css_pipe *css_pipe;
1977
1978         if (!css->streaming)
1979                 return ERR_PTR(-EPROTO);
1980
1981         r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
1982         if (r < 0)
1983                 return ERR_PTR(r);
1984
1985         evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1986                   IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1987
1988         switch (evtype) {
1989         case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1990         case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1991         case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1992         case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1993                 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1994                         IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1995                 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1996                         IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1997                 queue = evtype_to_queue[evtype];
1998                 qid = imgu_css_queues[queue].qid;
1999
2000                 if (pipe >= IMGU_MAX_PIPE_NUM) {
2001                         dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2002                         return ERR_PTR(-EIO);
2003                 }
2004
2005                 if (qid >= IMGU_ABI_QUEUE_NUM) {
2006                         dev_err(css->dev, "Invalid qid: %i\n", qid);
2007                         return ERR_PTR(-EIO);
2008                 }
2009                 css_pipe = &css->pipes[pipe];
2010                 dev_dbg(css->dev,
2011                         "event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2012                         event, queue, pipe, pipeid);
2013
2014                 r = imgu_css_dequeue_data(css, qid, &daddr);
2015                 if (r < 0) {
2016                         dev_err(css->dev, "failed to dequeue buffer\n");
2017                         /* Force real error, not -EBUSY */
2018                         return ERR_PTR(-EIO);
2019                 }
2020
2021                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2022                                         IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2023                 if (r < 0) {
2024                         dev_err(css->dev, "failed to queue event\n");
2025                         return ERR_PTR(-EIO);
2026                 }
2027
2028                 spin_lock(&css_pipe->qlock);
2029                 if (list_empty(&css_pipe->queue[queue].bufs)) {
2030                         spin_unlock(&css_pipe->qlock);
2031                         dev_err(css->dev, "event on empty queue\n");
2032                         return ERR_PTR(-EIO);
2033                 }
2034                 b = list_first_entry(&css_pipe->queue[queue].bufs,
2035                                      struct imgu_css_buffer, list);
2036                 if (queue != b->queue ||
2037                     daddr != css_pipe->abi_buffers
2038                         [b->queue][b->queue_pos].daddr) {
2039                         spin_unlock(&css_pipe->qlock);
2040                         dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2041                         return ERR_PTR(-EIO);
2042                 }
2043
2044                 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2045                 b->pipe = pipe;
2046                 b->state = IPU3_CSS_BUFFER_DONE;
2047                 list_del(&b->list);
2048                 spin_unlock(&css_pipe->qlock);
2049                 break;
2050         case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2051                 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2052                         IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2053                 if (pipe >= IMGU_MAX_PIPE_NUM) {
2054                         dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2055                         return ERR_PTR(-EIO);
2056                 }
2057
2058                 css_pipe = &css->pipes[pipe];
2059                 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2060                         event, pipe);
2061                 break;
2062         case IMGU_ABI_EVTTYPE_TIMER:
2063                 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2064                 if (r < 0)
2065                         return ERR_PTR(r);
2066
2067                 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2068                     IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2069                         dev_dbg(css->dev, "event: timer\n");
2070                 else
2071                         dev_warn(css->dev, "half of timer event missing\n");
2072                 break;
2073         case IMGU_ABI_EVTTYPE_FW_WARNING:
2074                 dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2075                 break;
2076         case IMGU_ABI_EVTTYPE_FW_ASSERT:
2077                 dev_err(css->dev,
2078                         "event: firmware assert 0x%x module_id %i line_no %i\n",
2079                         event,
2080                         (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2081                         IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2082                         swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2083                                IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2084                 break;
2085         default:
2086                 dev_warn(css->dev, "received unknown event 0x%x\n", event);
2087         }
2088
2089         return b;
2090 }
2091
2092 /*
2093  * Get a new set of parameters from pool and initialize them based on
2094  * the parameters params, gdc, and obgrid. Any of these may be NULL,
2095  * in which case the previously set parameters are used.
2096  * If parameters haven't been set previously, initialize from scratch.
2097  *
2098  * Return index to css->parameter_set_info which has the newly created
2099  * parameters or negative value on error.
2100  */
2101 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2102                             struct ipu3_uapi_params *set_params)
2103 {
2104         static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2105         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2106         const int stage = 0;
2107         const struct imgu_fw_info *bi;
2108         int obgrid_size;
2109         unsigned int stripes, i;
2110         struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2111
2112         /* Destination buffers which are filled here */
2113         struct imgu_abi_parameter_set_info *param_set;
2114         struct imgu_abi_acc_param *acc = NULL;
2115         struct imgu_abi_gdc_warp_param *gdc = NULL;
2116         struct ipu3_uapi_obgrid_param *obgrid = NULL;
2117         const struct imgu_css_map *map;
2118         void *vmem0 = NULL;
2119         void *dmem0 = NULL;
2120
2121         enum imgu_abi_memories m;
2122         int r = -EBUSY;
2123
2124         if (!css->streaming)
2125                 return -EPROTO;
2126
2127         dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2128
2129         bi = &css->fwp->binary_header[css_pipe->bindex];
2130         obgrid_size = imgu_css_fw_obgrid_size(bi);
2131         stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2132
2133         imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2134         param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2135                                        0)->vaddr;
2136
2137         /* Get a new acc only if new parameters given, or none yet */
2138         map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2139         if (set_params || !map->vaddr) {
2140                 imgu_css_pool_get(&css_pipe->pool.acc);
2141                 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2142                 acc = map->vaddr;
2143         }
2144
2145         /* Get new VMEM0 only if needed, or none yet */
2146         m = IMGU_ABI_MEM_ISP_VMEM0;
2147         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2148         if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2149                                            set_params->use.tnr3_vmem_params ||
2150                                            set_params->use.xnr3_vmem_params))) {
2151                 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2152                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2153                 vmem0 = map->vaddr;
2154         }
2155
2156         /* Get new DMEM0 only if needed, or none yet */
2157         m = IMGU_ABI_MEM_ISP_DMEM0;
2158         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2159         if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2160                                            set_params->use.xnr3_dmem_params))) {
2161                 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2162                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2163                 dmem0 = map->vaddr;
2164         }
2165
2166         /* Configure acc parameter cluster */
2167         if (acc) {
2168                 /* get acc_old */
2169                 map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2170                 /* user acc */
2171                 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2172                         set_params ? &set_params->acc_param : NULL);
2173                 if (r < 0)
2174                         goto fail;
2175         }
2176
2177         /* Configure late binding parameters */
2178         if (vmem0) {
2179                 m = IMGU_ABI_MEM_ISP_VMEM0;
2180                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2181                 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2182                                        map->vaddr, set_params);
2183                 if (r < 0)
2184                         goto fail;
2185         }
2186
2187         if (dmem0) {
2188                 m = IMGU_ABI_MEM_ISP_DMEM0;
2189                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2190                 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2191                                        map->vaddr, set_params);
2192                 if (r < 0)
2193                         goto fail;
2194         }
2195
2196         /* Get a new gdc only if a new gdc is given, or none yet */
2197         if (bi->info.isp.sp.enable.dvs_6axis) {
2198                 unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2199                 unsigned int g = IPU3_CSS_RECT_GDC;
2200                 unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2201
2202                 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2203                 if (!map->vaddr) {
2204                         imgu_css_pool_get(&css_pipe->pool.gdc);
2205                         map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2206                         gdc = map->vaddr;
2207                         imgu_css_cfg_gdc_table(map->vaddr,
2208                                 css_pipe->aux_frames[a].bytesperline /
2209                                 css_pipe->aux_frames[a].bytesperpixel,
2210                                 css_pipe->aux_frames[a].height,
2211                                 css_pipe->rect[g].width,
2212                                 css_pipe->rect[g].height,
2213                                 css_pipe->rect[e].width,
2214                                 css_pipe->rect[e].height);
2215                 }
2216         }
2217
2218         /* Get a new obgrid only if a new obgrid is given, or none yet */
2219         map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2220         if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2221                 imgu_css_pool_get(&css_pipe->pool.obgrid);
2222                 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2223                 obgrid = map->vaddr;
2224
2225                 /* Configure optical black level grid (obgrid) */
2226                 if (set_params && set_params->use.obgrid_param)
2227                         for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2228                                 obgrid[i] = set_params->obgrid_param;
2229                 else
2230                         memset(obgrid, 0, obgrid_size);
2231         }
2232
2233         /* Configure parameter set info, queued to `queue_id' */
2234
2235         memset(param_set, 0, sizeof(*param_set));
2236         map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2237         param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2238
2239         map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2240         param_set->mem_map.dvs_6axis_params_y = map->daddr;
2241
2242         for (i = 0; i < stripes; i++) {
2243                 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2244                 param_set->mem_map.obgrid_tbl[i] =
2245                         map->daddr + (obgrid_size / stripes) * i;
2246         }
2247
2248         for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2249                 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2250                 param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2251         }
2252
2253         /* Then queue the new parameter buffer */
2254         map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2255         r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2256         if (r < 0)
2257                 goto fail;
2258
2259         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2260                                 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2261                                                                queue_id));
2262         if (r < 0)
2263                 goto fail_no_put;
2264
2265         /* Finally dequeue all old parameter buffers */
2266
2267         do {
2268                 u32 daddr;
2269
2270                 r = imgu_css_dequeue_data(css, queue_id, &daddr);
2271                 if (r == -EBUSY)
2272                         break;
2273                 if (r)
2274                         goto fail_no_put;
2275                 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2276                                         IMGU_ABI_EVENT_BUFFER_DEQUEUED
2277                                         (queue_id));
2278                 if (r < 0) {
2279                         dev_err(css->dev, "failed to queue parameter event\n");
2280                         goto fail_no_put;
2281                 }
2282         } while (1);
2283
2284         return 0;
2285
2286 fail:
2287         /*
2288          * A failure, most likely the parameter queue was full.
2289          * Return error but continue streaming. User can try submitting new
2290          * parameters again later.
2291          */
2292
2293         imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2294         if (acc)
2295                 imgu_css_pool_put(&css_pipe->pool.acc);
2296         if (gdc)
2297                 imgu_css_pool_put(&css_pipe->pool.gdc);
2298         if (obgrid)
2299                 imgu_css_pool_put(&css_pipe->pool.obgrid);
2300         if (vmem0)
2301                 imgu_css_pool_put(
2302                         &css_pipe->pool.binary_params_p
2303                         [IMGU_ABI_MEM_ISP_VMEM0]);
2304         if (dmem0)
2305                 imgu_css_pool_put(
2306                         &css_pipe->pool.binary_params_p
2307                         [IMGU_ABI_MEM_ISP_DMEM0]);
2308
2309 fail_no_put:
2310         return r;
2311 }
2312
2313 int imgu_css_irq_ack(struct imgu_css *css)
2314 {
2315         static const int NUM_SWIRQS = 3;
2316         struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2317         void __iomem *const base = css->base;
2318         u32 irq_status[IMGU_IRQCTRL_NUM];
2319         int i;
2320
2321         u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2322
2323         writel(imgu_status, base + IMGU_REG_INT_STATUS);
2324         for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2325                 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2326
2327         for (i = 0; i < NUM_SWIRQS; i++) {
2328                 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2329                         /* SP SW interrupt */
2330                         u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2331                                         bi->info.sp.output);
2332                         u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2333                                         bi->info.sp.output + 4 + 4 * i);
2334
2335                         dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2336                                 __func__, i, cnt, val);
2337                 }
2338         }
2339
2340         for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2341                 if (irq_status[i]) {
2342                         writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2343                         /* Wait for write to complete */
2344                         readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2345                 }
2346
2347         dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2348                 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2349                 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2350
2351         if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2352                 return -ENOMSG;
2353
2354         return 0;
2355 }