libavfilter/vf_scale_vt.c

   1 /*
   2  * Copyright (c) 2023 Zhao Zhili <zhilizhao@tencent.com>
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include <VideoToolbox/VideoToolbox.h>
  22
  23 #include "libavutil/hwcontext.h"
  24 #include "libavutil/hwcontext_videotoolbox.h"
  25 #include "libavutil/opt.h"
  26 #include "libavutil/pixdesc.h"
  27 #include "internal.h"
  28 #include "scale_eval.h"
  29 #include "video.h"
  30
  31 typedef struct ScaleVtContext {
  32     AVClass *class;
  33
  34     VTPixelTransferSessionRef transfer;
  35     int output_width;
  36     int output_height;
  37     char *w_expr;
  38     char *h_expr;
  39
  40     enum AVColorPrimaries colour_primaries;
  41     enum AVColorTransferCharacteristic colour_transfer;
  42     enum AVColorSpace colour_matrix;
  43     char *colour_primaries_string;
  44     char *colour_transfer_string;
  45     char *colour_matrix_string;
  46 } ScaleVtContext;
  47
  48 static av_cold int scale_vt_init(AVFilterContext *avctx)
  49 {
  50     ScaleVtContext *s = avctx->priv;
  51     int ret;
  52     CFStringRef value;
  53
  54     ret = VTPixelTransferSessionCreate(kCFAllocatorDefault, &s->transfer);
  55     if (ret != noErr) {
  56         av_log(avctx, AV_LOG_ERROR, "transfer session create failed, %d\n", ret);
  57         return AVERROR_EXTERNAL;
  58     }
  59
  60 #define STRING_OPTION(var_name, func_name, default_value)                \
  61     do {                                                                 \
  62         if (s->var_name##_string) {                                      \
  63             int var = av_##func_name##_from_name(s->var_name##_string);  \
  64             if (var < 0) {                                               \
  65                 av_log(avctx, AV_LOG_ERROR, "Invalid %s.\n", #var_name); \
  66                 return AVERROR(EINVAL);                                  \
  67             }                                                            \
  68             s->var_name = var;                                           \
  69         } else {                                                         \
  70             s->var_name = default_value;                                 \
  71         }                                                                \
  72     } while (0)
  73
  74     STRING_OPTION(colour_primaries, color_primaries, AVCOL_PRI_UNSPECIFIED);
  75     STRING_OPTION(colour_transfer,  color_transfer,  AVCOL_TRC_UNSPECIFIED);
  76     STRING_OPTION(colour_matrix,    color_space,     AVCOL_SPC_UNSPECIFIED);
  77
  78     if (s->colour_primaries != AVCOL_PRI_UNSPECIFIED) {
  79         value = av_map_videotoolbox_color_primaries_from_av(s->colour_primaries);
  80         if (!value) {
  81             av_log(avctx, AV_LOG_ERROR,
  82                    "Doesn't support converting to colour primaries %s\n",
  83                    s->colour_primaries_string);
  84             return AVERROR(ENOTSUP);
  85         }
  86         VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationColorPrimaries, value);
  87     }
  88
  89     if (s->colour_transfer != AVCOL_TRC_UNSPECIFIED) {
  90         value = av_map_videotoolbox_color_trc_from_av(s->colour_transfer);
  91         if (!value) {
  92             av_log(avctx, AV_LOG_ERROR,
  93                    "Doesn't support converting to trc %s\n",
  94                    s->colour_transfer_string);
  95             return AVERROR(ENOTSUP);
  96         }
  97         VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationTransferFunction, value);
  98     }
  99
 100     if (s->colour_matrix != AVCOL_SPC_UNSPECIFIED) {
 101         value = av_map_videotoolbox_color_matrix_from_av(s->colour_matrix);
 102         if (!value) {
 103             av_log(avctx, AV_LOG_ERROR,
 104                    "Doesn't support converting to colorspace %s\n",
 105                    s->colour_matrix_string);
 106             return AVERROR(ENOTSUP);
 107         }
 108         VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationYCbCrMatrix, value);
 109     }
 110
 111     return 0;
 112 }
 113
 114 static av_cold void scale_vt_uninit(AVFilterContext *avctx)
 115 {
 116     ScaleVtContext *s = avctx->priv;
 117
 118     if (s->transfer) {
 119         VTPixelTransferSessionInvalidate(s->transfer);
 120         CFRelease(s->transfer);
 121         s->transfer = NULL;
 122     }
 123 }
 124
 125 static int scale_vt_filter_frame(AVFilterLink *link, AVFrame *in)
 126 {
 127     int ret;
 128     AVFilterContext *ctx = link->dst;
 129     ScaleVtContext *s = ctx->priv;
 130     AVFilterLink *outlink = ctx->outputs[0];
 131     CVPixelBufferRef src;
 132     CVPixelBufferRef dst;
 133
 134     AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
 135     if (!out) {
 136         ret = AVERROR(ENOMEM);
 137         goto fail;
 138     }
 139
 140     ret = av_frame_copy_props(out, in);
 141     if (ret < 0)
 142         goto fail;
 143
 144     av_reduce(&out->sample_aspect_ratio.num, &out->sample_aspect_ratio.den,
 145               (int64_t)in->sample_aspect_ratio.num * outlink->h * link->w,
 146               (int64_t)in->sample_aspect_ratio.den * outlink->w * link->h,
 147               INT_MAX);
 148     if (s->colour_primaries != AVCOL_PRI_UNSPECIFIED)
 149         out->color_primaries = s->colour_primaries;
 150     if (s->colour_transfer != AVCOL_TRC_UNSPECIFIED)
 151         out->color_trc = s->colour_transfer;
 152     if (s->colour_matrix != AVCOL_SPC_UNSPECIFIED)
 153         out->colorspace = s->colour_matrix;
 154
 155     src = (CVPixelBufferRef)in->data[3];
 156     dst = (CVPixelBufferRef)out->data[3];
 157     ret = VTPixelTransferSessionTransferImage(s->transfer, src, dst);
 158     if (ret != noErr) {
 159         av_log(ctx, AV_LOG_ERROR, "transfer image failed, %d\n", ret);
 160         ret = AVERROR_EXTERNAL;
 161         goto fail;
 162     }
 163
 164     av_frame_free(&in);
 165
 166     return ff_filter_frame(outlink, out);
 167
 168 fail:
 169     av_frame_free(&in);
 170     av_frame_free(&out);
 171     return ret;
 172 }
 173
 174 static int scale_vt_config_output(AVFilterLink *outlink)
 175 {
 176     int err;
 177     AVFilterContext *avctx = outlink->src;
 178     ScaleVtContext *s  = avctx->priv;
 179     AVFilterLink *inlink = outlink->src->inputs[0];
 180     AVHWFramesContext *hw_frame_ctx_in;
 181     AVHWFramesContext *hw_frame_ctx_out;
 182
 183     err = ff_scale_eval_dimensions(s, s->w_expr, s->h_expr, inlink, outlink,
 184                                    &s->output_width,
 185                                    &s->output_height);
 186     if (err < 0)
 187         return err;
 188
 189     outlink->w = s->output_width;
 190     outlink->h = s->output_height;
 191
 192     if (inlink->sample_aspect_ratio.num) {
 193         AVRational r = {outlink->h * inlink->w, outlink->w * inlink->h};
 194         outlink->sample_aspect_ratio = av_mul_q(r, inlink->sample_aspect_ratio);
 195     } else {
 196         outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
 197     }
 198
 199     hw_frame_ctx_in = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
 200
 201     av_buffer_unref(&outlink->hw_frames_ctx);
 202     outlink->hw_frames_ctx = av_hwframe_ctx_alloc(hw_frame_ctx_in->device_ref);
 203     hw_frame_ctx_out = (AVHWFramesContext *)outlink->hw_frames_ctx->data;
 204     hw_frame_ctx_out->format = AV_PIX_FMT_VIDEOTOOLBOX;
 205     hw_frame_ctx_out->sw_format = hw_frame_ctx_in->sw_format;
 206     hw_frame_ctx_out->width = outlink->w;
 207     hw_frame_ctx_out->height = outlink->h;
 208
 209     err = ff_filter_init_hw_frames(avctx, outlink, 1);
 210     if (err < 0)
 211         return err;
 212
 213     err = av_hwframe_ctx_init(outlink->hw_frames_ctx);
 214     if (err < 0) {
 215         av_log(avctx, AV_LOG_ERROR,
 216                "Failed to init videotoolbox frame context, %s\n",
 217                av_err2str(err));
 218         return err;
 219     }
 220
 221     return 0;
 222 }
 223
 224 #define OFFSET(x) offsetof(ScaleVtContext, x)
 225 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
 226 static const AVOption scale_vt_options[] = {
 227     { "w", "Output video width",
 228             OFFSET(w_expr), AV_OPT_TYPE_STRING, {.str = "iw"}, .flags = FLAGS },
 229     { "h", "Output video height",
 230             OFFSET(h_expr), AV_OPT_TYPE_STRING, {.str = "ih"}, .flags = FLAGS },
 231     { "color_matrix", "Output colour matrix coefficient set",
 232             OFFSET(colour_matrix_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
 233     { "color_primaries", "Output colour primaries",
 234             OFFSET(colour_primaries_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
 235     { "color_transfer", "Output colour transfer characteristics",
 236             OFFSET(colour_transfer_string),  AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS },
 237     { NULL },
 238 };
 239
 240 AVFILTER_DEFINE_CLASS(scale_vt);
 241
 242 static const AVFilterPad scale_vt_inputs[] = {
 243     {
 244         .name         = "default",
 245         .type         = AVMEDIA_TYPE_VIDEO,
 246         .filter_frame = &scale_vt_filter_frame,
 247     },
 248 };
 249
 250 static const AVFilterPad scale_vt_outputs[] = {
 251     {
 252         .name = "default",
 253         .type = AVMEDIA_TYPE_VIDEO,
 254         .config_props = &scale_vt_config_output,
 255     },
 256 };
 257
 258 const AVFilter ff_vf_scale_vt = {
 259     .name           = "scale_vt",
 260     .description    = NULL_IF_CONFIG_SMALL("Scale Videotoolbox frames"),
 261     .priv_size      = sizeof(ScaleVtContext),
 262     .init           = scale_vt_init,
 263     .uninit         = scale_vt_uninit,
 264     FILTER_INPUTS(scale_vt_inputs),
 265     FILTER_OUTPUTS(scale_vt_outputs),
 266     FILTER_SINGLE_PIXFMT(AV_PIX_FMT_VIDEOTOOLBOX),
 267     .priv_class     = &scale_vt_class,
 268     .flags          = AVFILTER_FLAG_HWDEVICE,
 269     .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 270 };