From f8016687a4fc65d7266cf3ac712d73e8e09eaab6 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sun, 30 Jan 2022 03:11:37 +0900 Subject: [PATCH] wic: Add support for JPEG and PNG decoding Adding Windows Imaging Component (WIC) plugin with JPEG/PNG decoding support. Part-of: --- subprojects/gst-plugins-bad/meson_options.txt | 1 + subprojects/gst-plugins-bad/sys/meson.build | 1 + .../gst-plugins-bad/sys/wic/gstwicdecoder.cpp | 281 +++++++++++ .../gst-plugins-bad/sys/wic/gstwicdecoder.h | 64 +++ .../sys/wic/gstwicimagingfactory.cpp | 192 ++++++++ .../gst-plugins-bad/sys/wic/gstwicimagingfactory.h | 39 ++ .../gst-plugins-bad/sys/wic/gstwicjpegdec.cpp | 544 +++++++++++++++++++++ .../gst-plugins-bad/sys/wic/gstwicjpegdec.h | 31 ++ .../gst-plugins-bad/sys/wic/gstwicpngdec.cpp | 315 ++++++++++++ subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.h | 31 ++ .../gst-plugins-bad/sys/wic/gstwicutils.cpp | 123 +++++ subprojects/gst-plugins-bad/sys/wic/gstwicutils.h | 53 ++ subprojects/gst-plugins-bad/sys/wic/meson.build | 79 +++ subprojects/gst-plugins-bad/sys/wic/plugin.cpp | 77 +++ 14 files changed, 1831 insertions(+) create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.cpp create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.h create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.cpp create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.h create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.cpp create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.h create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.cpp create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.h create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicutils.cpp create mode 100644 subprojects/gst-plugins-bad/sys/wic/gstwicutils.h create mode 100644 subprojects/gst-plugins-bad/sys/wic/meson.build create mode 100644 subprojects/gst-plugins-bad/sys/wic/plugin.cpp diff --git a/subprojects/gst-plugins-bad/meson_options.txt b/subprojects/gst-plugins-bad/meson_options.txt index c68eb75..5ad0d18 100644 --- a/subprojects/gst-plugins-bad/meson_options.txt +++ b/subprojects/gst-plugins-bad/meson_options.txt @@ -170,6 +170,7 @@ option('webp', type : 'feature', value : 'auto', description : 'WebP image codec option('webrtc', type : 'feature', value : 'auto', description : 'WebRTC audio/video network bin plugin') option('webrtcdsp', type : 'feature', value : 'auto', description : 'Plugin with various audio filters provided by the WebRTC audio processing library') option('wildmidi', type : 'feature', value : 'auto', description : 'WildMidi midi soft synth plugin') +option('wic', type : 'feature', value : 'auto', description : 'Windows Imaging Component plugin') option('winks', type : 'feature', value : 'auto', description : 'Windows Kernel Streaming video source plugin') option('winscreencap', type : 'feature', value : 'auto', description : 'Windows Screen Capture video source plugin') option('x265', type : 'feature', value : 'auto', description : 'HEVC/H.265 video encoder plugin (GPL - only built if gpl option is also enabled!)') diff --git a/subprojects/gst-plugins-bad/sys/meson.build b/subprojects/gst-plugins-bad/sys/meson.build index 2b7d2f1..01635bb 100644 --- a/subprojects/gst-plugins-bad/sys/meson.build +++ b/subprojects/gst-plugins-bad/sys/meson.build @@ -24,5 +24,6 @@ subdir('v4l2codecs') subdir('va') subdir('wasapi') subdir('wasapi2') +subdir('wic') subdir('winks') subdir('winscreencap') diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.cpp b/subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.cpp new file mode 100644 index 0000000..04ce8f4 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.cpp @@ -0,0 +1,281 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstwicdecoder.h" + +#include +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + + +GST_DEBUG_CATEGORY_STATIC (gst_wic_decoder_debug); +#define GST_CAT_DEFAULT gst_wic_decoder_debug + +struct _GstWicDecoderPrivate +{ + GstWicImagingFactory *factory; + IStream *stream; +}; + +static gboolean gst_wic_decoder_open (GstVideoDecoder * decoder); +static gboolean gst_wic_decoder_close (GstVideoDecoder * decoder); +static gboolean gst_wic_decoder_stop (GstVideoDecoder * decoder); +static gboolean gst_wic_decoder_set_format (GstVideoDecoder * decoder, + GstVideoCodecState * state); +static GstFlowReturn gst_wic_decoder_handle_frame (GstVideoDecoder * decoder, + GstVideoCodecFrame * frame); + +#define gst_wic_decoder_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE (GstWicDecoder, gst_wic_decoder, + GST_TYPE_VIDEO_DECODER); + +static void +gst_wic_decoder_class_init (GstWicDecoderClass * klass) +{ + GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass); + + decoder_class->open = GST_DEBUG_FUNCPTR (gst_wic_decoder_open); + decoder_class->close = GST_DEBUG_FUNCPTR (gst_wic_decoder_close); + decoder_class->stop = GST_DEBUG_FUNCPTR (gst_wic_decoder_stop); + decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_wic_decoder_set_format); + decoder_class->handle_frame = + GST_DEBUG_FUNCPTR (gst_wic_decoder_handle_frame); + + GST_DEBUG_CATEGORY_INIT (gst_wic_decoder_debug, + "wicdecoder", 0, "wicdecoder"); + + gst_type_mark_as_plugin_api (GST_TYPE_WIC_DECODER, (GstPluginAPIFlags) 0); +} + +static void +gst_wic_decoder_init (GstWicDecoder * self) +{ + self->priv = + (GstWicDecoderPrivate *) gst_wic_decoder_get_instance_private (self); +} + +static gboolean +gst_wic_decoder_open (GstVideoDecoder * decoder) +{ + GstWicDecoder *self = GST_WIC_DECODER (decoder); + GstWicDecoderClass *klass = GST_WIC_DECODER_GET_CLASS (self); + GstWicDecoderPrivate *priv = self->priv; + HRESULT hr; + ComPtr < IStream > stream; + + priv->factory = gst_wic_imaging_factory_new (); + if (!priv->factory) { + GST_ERROR_OBJECT (self, "Failed to create factory"); + return FALSE; + } + + hr = gst_wic_imaging_factory_check_codec_support (priv->factory, + TRUE, klass->codec_id); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Codec is not supported, hr: 0x%x", (guint) hr); + goto error; + } + + hr = CreateStreamOnHGlobal (nullptr, TRUE, &stream); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create IStream object"); + goto error; + } + + priv->stream = stream.Detach (); + + return TRUE; + +error: + GST_WIC_CLEAR_COM (priv->stream); + gst_clear_object (&priv->factory); + + return FALSE; +} + +static gboolean +gst_wic_decoder_close (GstVideoDecoder * decoder) +{ + GstWicDecoder *self = GST_WIC_DECODER (decoder); + GstWicDecoderPrivate *priv = self->priv; + + GST_WIC_CLEAR_COM (priv->stream); + gst_clear_object (&priv->factory); + + return TRUE; +} + +static gboolean +gst_wic_decoder_stop (GstVideoDecoder * decoder) +{ + GstWicDecoder *self = GST_WIC_DECODER (decoder); + + g_clear_pointer (&self->input_state, gst_video_codec_state_unref); + + return TRUE; +} + +static gboolean +gst_wic_decoder_set_format (GstVideoDecoder * decoder, + GstVideoCodecState * state) +{ + GstWicDecoder *self = GST_WIC_DECODER (decoder); + GstWicDecoderClass *klass = GST_WIC_DECODER_GET_CLASS (self); + + self->input_state = gst_video_codec_state_ref (state); + + if (klass->set_format) + return klass->set_format (self, state); + + return TRUE; +} + +static gboolean +gst_wic_decoder_upload (GstWicDecoder * self, GstBuffer * buffer) +{ + GstWicDecoderPrivate *priv = self->priv; + IStream *stream = priv->stream; + LARGE_INTEGER pos; + ULARGE_INTEGER size; + HRESULT hr; + GstMapInfo info; + + if (!gst_buffer_map (buffer, &info, GST_MAP_READ)) { + GST_ELEMENT_ERROR (self, + RESOURCE, READ, ("Unable to map buffer"), (nullptr)); + return FALSE; + } + + size.QuadPart = info.size; + hr = stream->SetSize (size); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, + "Failed to set size to %" G_GSIZE_FORMAT, info.size); + goto read_write_error; + } + + pos.QuadPart = 0; + hr = stream->Seek (pos, STREAM_SEEK_SET, nullptr); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to seek IStream"); + gst_buffer_unmap (buffer, &info); + goto read_write_error; + } + + hr = stream->Write (info.data, info.size, nullptr); + + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to write data into IStream"); + goto read_write_error; + } + + hr = stream->Seek (pos, STREAM_SEEK_SET, nullptr); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to seek IStream"); + goto read_write_error; + } + + gst_buffer_unmap (buffer, &info); + + return TRUE; + +read_write_error: + { + gchar *error_text = g_win32_error_message ((guint) hr); + + GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ_WRITE, + ("Failed to read/write stream for decoding"), + ("HRSULT: 0x%x (%s)", (guint) hr, GST_STR_NULL (error_text))); + g_free (error_text); + + gst_buffer_unmap (buffer, &info); + + return FALSE; + } +} + +static gboolean +gst_wic_decoder_decode (GstWicDecoder * self, IWICBitmapDecoder ** decoder) +{ + GstWicDecoderClass *klass = GST_WIC_DECODER_GET_CLASS (self); + GstWicDecoderPrivate *priv = self->priv; + IStream *stream = priv->stream; + IWICImagingFactory *factory; + HRESULT hr; + ComPtr < IWICBitmapDecoder > handle; + + factory = gst_wic_imaging_factory_get_handle (priv->factory); + hr = factory->CreateDecoder (klass->codec_id, nullptr, &handle); + if (FAILED (hr)) { + GST_ELEMENT_ERROR (self, STREAM, DECODE, ("Unable to create decoder"), + ("IWICImagingFactory::IWICBitmapDecoder returned hr 0x%x", (guint) hr)); + return FALSE; + } + + hr = handle->Initialize (stream, WICDecodeMetadataCacheOnLoad); + if (FAILED (hr)) { + GST_ELEMENT_ERROR (self, STREAM, DECODE, ("Unable initialize decoder"), + ("IWICBitmapDecoder::Initialize returned hr 0x%x", (guint) hr)); + return FALSE; + } + + *decoder = handle.Detach (); + + return TRUE; +} + +static GstFlowReturn +gst_wic_decoder_handle_frame (GstVideoDecoder * decoder, + GstVideoCodecFrame * frame) +{ + GstWicDecoder *self = GST_WIC_DECODER (decoder); + GstWicDecoderClass *klass = GST_WIC_DECODER_GET_CLASS (self); + GstWicDecoderPrivate *priv = self->priv; + ComPtr < IWICBitmapDecoder > handle; + ComPtr < IWICBitmapFrameDecode > decode_frame; + HRESULT hr; + IWICImagingFactory *factory; + + factory = gst_wic_imaging_factory_get_handle (priv->factory); + + if (!gst_wic_decoder_upload (self, frame->input_buffer)) { + gst_video_decoder_release_frame (decoder, frame); + return GST_FLOW_ERROR; + } + + if (!gst_wic_decoder_decode (self, &handle)) { + gst_video_decoder_release_frame (decoder, frame); + return GST_FLOW_ERROR; + } + + hr = handle->GetFrame (0, &decode_frame); + if (FAILED (hr)) { + gst_video_decoder_release_frame (decoder, frame); + + GST_ELEMENT_ERROR (self, STREAM, DECODE, ("Failed to decode"), + ("IWICBitmapDecoder::GetFrame returned hr 0x%x", (guint) hr)); + + return GST_FLOW_ERROR; + } + + return klass->process_output (self, factory, decode_frame.Get (), frame); +} diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.h b/subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.h new file mode 100644 index 0000000..99b69fc --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicdecoder.h @@ -0,0 +1,64 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include "gstwicimagingfactory.h" + +G_BEGIN_DECLS + +#define GST_TYPE_WIC_DECODER (gst_wic_decoder_get_type()) +#define GST_WIC_DECODER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_WIC_DECODER,GstWicDecoder)) +#define GST_WIC_DECODER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_WIC_DECODER,GstWicDecoderClass)) +#define GST_WIC_DECODER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_WIC_DECODER,GstWicDecoderClass)) +#define GST_IS_WIC_DECODER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_WIC_DECODER)) +#define GST_IS_WIC_DECODER_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_WIC_DECODER)) + +typedef struct _GstWicDecoder GstWicDecoder; +typedef struct _GstWicDecoderClass GstWicDecoderClass; +typedef struct _GstWicDecoderPrivate GstWicDecoderPrivate; + +struct _GstWicDecoder +{ + GstVideoDecoder parent; + + GstVideoCodecState *input_state; + + GstWicDecoderPrivate *priv; +}; + +struct _GstWicDecoderClass +{ + GstVideoDecoderClass parent_class; + GUID codec_id; + + gboolean (*set_format) (GstWicDecoder * decoder, + GstVideoCodecState * state); + + GstFlowReturn (*process_output) (GstWicDecoder * decoder, + IWICImagingFactory * factory, + IWICBitmapFrameDecode * decode_frame, + GstVideoCodecFrame * frame); +}; + +GType gst_wic_decoder_get_type (void); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.cpp b/subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.cpp new file mode 100644 index 0000000..a14dbd9 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.cpp @@ -0,0 +1,192 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstwicimagingfactory.h" +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +GST_DEBUG_CATEGORY_STATIC (gst_wic_imaging_factory_debug); +#define GST_CAT_DEFAULT gst_wic_imaging_factory_debug + +struct _GstWicImagingFactory +{ + GstObject parent; + + IWICImagingFactory *handle; + + GThread *thread; + GMutex lock; + GCond cond; + GMainContext *context; + GMainLoop *loop; +}; + +static void gst_wic_imaging_factory_constructed (GObject * object); +static void gst_wic_imaging_factory_finalize (GObject * object); +static gpointer gst_wic_imaging_factory_func (GstWicImagingFactory * self); + +#define gst_wic_imaging_factory_parent_class parent_class +G_DEFINE_TYPE (GstWicImagingFactory, gst_wic_imaging_factory, GST_TYPE_OBJECT); + +static void +gst_wic_imaging_factory_class_init (GstWicImagingFactoryClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->constructed = gst_wic_imaging_factory_constructed; + object_class->finalize = gst_wic_imaging_factory_finalize; +} + +static void +gst_wic_imaging_factory_init (GstWicImagingFactory * self) +{ + g_mutex_init (&self->lock); + g_cond_init (&self->cond); + + self->context = g_main_context_new (); + self->loop = g_main_loop_new (self->context, FALSE); +} + +static void +gst_wic_imaging_factory_constructed (GObject * object) +{ + GstWicImagingFactory *self = GST_WIC_IMAGING_FACTORY (object); + + g_mutex_lock (&self->lock); + self->thread = g_thread_new ("GstWicImagingFactory", + (GThreadFunc) gst_wic_imaging_factory_func, self); + while (!g_main_loop_is_running (self->loop)) + g_cond_wait (&self->cond, &self->lock); + g_mutex_unlock (&self->lock); + + G_OBJECT_CLASS (parent_class)->constructed (object); +} + +static void +gst_wic_imaging_factory_finalize (GObject * object) +{ + GstWicImagingFactory *self = GST_WIC_IMAGING_FACTORY (object); + + if (self->loop) { + g_main_loop_quit (self->loop); + g_thread_join (self->thread); + + g_main_loop_unref (self->loop); + g_main_context_unref (self->context); + } + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static gboolean +loop_running_cb (GstWicImagingFactory * self) +{ + g_mutex_lock (&self->lock); + g_cond_signal (&self->cond); + g_mutex_unlock (&self->lock); + + return G_SOURCE_REMOVE; +} + +static gpointer +gst_wic_imaging_factory_func (GstWicImagingFactory * self) +{ + HRESULT hr; + ComPtr < IWICImagingFactory > factory; + GSource *idle_source; + + CoInitializeEx (nullptr, COINIT_MULTITHREADED); + + g_main_context_push_thread_default (self->context); + + idle_source = g_idle_source_new (); + g_source_set_callback (idle_source, + (GSourceFunc) loop_running_cb, self, nullptr); + g_source_attach (idle_source, self->context); + g_source_unref (idle_source); + + hr = CoCreateInstance (CLSID_WICImagingFactory, nullptr, + CLSCTX_INPROC_SERVER, IID_PPV_ARGS (&factory)); + + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create factory handle, hr: 0x%x", + (guint) hr); + } else { + self->handle = factory.Detach (); + } + +run_loop: + g_main_loop_run (self->loop); + + if (self->handle) + self->handle->Release (); + + g_main_context_pop_thread_default (self->context); + CoUninitialize (); + + return nullptr; +} + +GstWicImagingFactory * +gst_wic_imaging_factory_new (void) +{ + GstWicImagingFactory *self; + + self = (GstWicImagingFactory *) g_object_new (GST_TYPE_WIC_IMAGING_FACTORY, + nullptr); + + if (!self->handle) { + gst_object_unref (self); + return nullptr; + } + + gst_object_ref_sink (self); + + return self; +} + +IWICImagingFactory * +gst_wic_imaging_factory_get_handle (GstWicImagingFactory * factory) +{ + g_return_val_if_fail (GST_IS_WIC_IMAGING_FACTORY (factory), nullptr); + + return factory->handle; +} + +HRESULT +gst_wic_imaging_factory_check_codec_support (GstWicImagingFactory * factory, + gboolean is_decoder, REFGUID codec_id) +{ + HRESULT hr = E_FAIL; + + g_return_val_if_fail (GST_IS_WIC_IMAGING_FACTORY (factory), E_FAIL); + + if (is_decoder) { + ComPtr < IWICBitmapDecoder > decoder; + hr = factory->handle->CreateDecoder (codec_id, nullptr, &decoder); + } else { + ComPtr < IWICBitmapEncoder > encoder; + hr = factory->handle->CreateEncoder (codec_id, nullptr, &encoder); + } + + return hr; +} diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.h b/subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.h new file mode 100644 index 0000000..4d5444c --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicimagingfactory.h @@ -0,0 +1,39 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include "gstwicutils.h" + +G_BEGIN_DECLS + +#define GST_TYPE_WIC_IMAGING_FACTORY (gst_wic_imaging_factory_get_type()) +G_DECLARE_FINAL_TYPE (GstWicImagingFactory, gst_wic_imaging_factory, + GST, WIC_IMAGING_FACTORY, GstObject); + +GstWicImagingFactory * gst_wic_imaging_factory_new (void); + +IWICImagingFactory * gst_wic_imaging_factory_get_handle (GstWicImagingFactory * factory); + +HRESULT gst_wic_imaging_factory_check_codec_support (GstWicImagingFactory * factory, + gboolean is_decoder, + REFGUID codec_id); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.cpp b/subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.cpp new file mode 100644 index 0000000..aa49ea0 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.cpp @@ -0,0 +1,544 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-wicjpegdec + * @title: wicjpegdec + * + * This element decodes JPEG compressed data into RAW video data. + * + * Since: 1.22 + * + */ + +#include "gstwicjpegdec.h" + +#include +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +GST_DEBUG_CATEGORY_STATIC (gst_wic_jpeg_dec_debug); +#define GST_CAT_DEFAULT gst_wic_jpeg_dec_debug + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("image/jpeg") + ); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ BGR, GRAY8, NV12, Y42B, Y444 }")) + ); + + +struct _GstWicJpegDec +{ + GstWicDecoder parent; + + WICBitmapPlaneDescription plane_desc[GST_VIDEO_MAX_PLANES]; + + GstVideoInfo info; +}; + +static gboolean gst_wic_jpeg_dec_set_format (GstWicDecoder * decoder, + GstVideoCodecState * state); +static GstFlowReturn gst_wic_jpeg_dec_process_output (GstWicDecoder * decoder, + IWICImagingFactory * factory, IWICBitmapFrameDecode * decode_frame, + GstVideoCodecFrame * frame); + +#define gst_wic_decoder_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstWicJpegDec, gst_wic_jpeg_dec, GST_TYPE_WIC_DECODER, + GST_DEBUG_CATEGORY_INIT (gst_wic_jpeg_dec_debug, + "wicjpegdec", 0, "wicjpegdec")); + +static void +gst_wic_jpeg_dec_class_init (GstWicJpegDecClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstWicDecoderClass *decoder_class = GST_WIC_DECODER_CLASS (klass); + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + + gst_element_class_set_static_metadata (element_class, + "Windows Imaging Component JPEG decoder", "Codec/Decoder/Image", + "Jpeg image decoder using Windows Imaging Component API", + "Seungha Yang "); + + decoder_class->codec_id = GUID_ContainerFormatJpeg; + decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_wic_jpeg_dec_set_format); + decoder_class->process_output = + GST_DEBUG_FUNCPTR (gst_wic_jpeg_dec_process_output); +} + +static void +gst_wic_jpeg_dec_init (GstWicJpegDec * self) +{ + gst_video_info_init (&self->info); + gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), TRUE); +} + +static gboolean +gst_wic_jpeg_dec_prepare_yuv_output (GstWicJpegDec * self, + WICJpegFrameHeader * hdr, + IWICBitmapSource * input, guint out_width, guint out_height, + IWICPlanarBitmapSourceTransform ** transform) +{ + ComPtr < IWICPlanarBitmapSourceTransform > tr; + + HRESULT hr; + BOOL is_supported = FALSE; + UINT32 supported_width = out_width; + UINT32 supported_height = out_height; + const WICPixelFormatGUID yuv_planar_formats[] = { + GUID_WICPixelFormat8bppY, + GUID_WICPixelFormat8bppCb, + GUID_WICPixelFormat8bppCr + }; + const WICPixelFormatGUID nv12_formats[] = { + GUID_WICPixelFormat8bppY, + GUID_WICPixelFormat16bppCbCr, + }; + const WICPixelFormatGUID *dst_formats = yuv_planar_formats; + guint n_planes = G_N_ELEMENTS (yuv_planar_formats); + GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN; + + switch (hdr->SampleFactors) { + case WIC_JPEG_SAMPLE_FACTORS_ONE: + /* GRAY */ + return FALSE; + case WIC_JPEG_SAMPLE_FACTORS_THREE_420: + /* NV12 is preferred over I420 on Windows, because I420 is not suppported + * by various Windows APIs, Specifically DXGI doesn't support I420 + * natively */ + format = GST_VIDEO_FORMAT_NV12; + dst_formats = nv12_formats; + n_planes = G_N_ELEMENTS (nv12_formats); + break; + case WIC_JPEG_SAMPLE_FACTORS_THREE_422: + format = GST_VIDEO_FORMAT_Y42B; + break; + case WIC_JPEG_SAMPLE_FACTORS_THREE_444: + format = GST_VIDEO_FORMAT_Y444; + break; + case WIC_JPEG_SAMPLE_FACTORS_THREE_440: + /* We don't support this format */ + return FALSE; + default: + return FALSE; + } + + hr = input->QueryInterface (IID_PPV_ARGS (&tr)); + if (FAILED (hr)) { + GST_TRACE_OBJECT (self, "IWICPlanarBitmapSourceTransform is not supported"); + return FALSE; + } + + hr = tr->DoesSupportTransform (&supported_width, + &supported_height, + WICBitmapTransformRotate0, + WICPlanarOptionsPreserveSubsampling, + dst_formats, self->plane_desc, n_planes, &is_supported); + + if (FAILED (hr) || !is_supported) { + GST_TRACE_OBJECT (self, "Transform is not supported"); + return FALSE; + } + + GST_LOG_OBJECT (self, "Transform supported %dx%d -> %dx%d", + out_width, out_height, supported_width, supported_height); + for (guint i = 0; i < n_planes; i++) { + GST_LOG_OBJECT (self, "Plane %d, %dx%d", i, + self->plane_desc[i].Width, self->plane_desc[i].Height); + } + + gst_video_info_set_format (&self->info, format, supported_width, + supported_height); + + *transform = tr.Detach (); + + return TRUE; +} + +static gboolean +gst_wic_jpeg_dec_prepare_rgb_output (GstWicJpegDec * self, + IWICImagingFactory * factory, IWICBitmapSource * input, + guint out_width, guint out_height, IWICBitmapSource ** source) +{ + WICPixelFormatGUID native_pixel_format; + GstVideoFormat native_format = GST_VIDEO_FORMAT_UNKNOWN; + HRESULT hr; + ComPtr < IWICBitmapSource > output; + + hr = input->GetPixelFormat (&native_pixel_format); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to query pixel format, hr: 0x%x", + (guint) hr); + return FALSE; + } + + /* native output formats are BGR, GRAY and CMYK but we don't support + * CMYK */ + if (!gst_wic_pixel_format_to_gst (native_pixel_format, &native_format) || + (native_format != GST_VIDEO_FORMAT_BGR && + native_format != GST_VIDEO_FORMAT_GRAY8)) { + ComPtr < IWICFormatConverter > conv; + + GST_LOG_OBJECT (self, + "Native format is not supported for output, needs conversion"); + + native_format = GST_VIDEO_FORMAT_BGR; + + if (!gst_wic_pixel_format_from_gst (native_format, &native_pixel_format)) { + GST_ERROR_OBJECT (self, "Failed to convert format to WIC"); + return FALSE; + } + + hr = factory->CreateFormatConverter (&conv); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create converter"); + return FALSE; + } + + hr = conv->Initialize (input, native_pixel_format, + WICBitmapDitherTypeNone, nullptr, 0.0f, WICBitmapPaletteTypeCustom); + + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to initialize converter, hr: 0x%x", + (guint) hr); + return FALSE; + } + + conv.As (&output); + } else { + output = input; + } + + gst_video_info_set_format (&self->info, native_format, out_width, out_height); + + *source = output.Detach (); + + return TRUE; +} + +static gboolean +gst_wic_jpeg_dec_fill_yuv_output (GstWicJpegDec * self, + IWICImagingFactory * factory, IWICPlanarBitmapSourceTransform * transform, + GstBuffer * buffer) +{ + ComPtr < IWICBitmap > bitmaps[GST_VIDEO_MAX_PLANES]; + ComPtr < IWICBitmapLock > locks[GST_VIDEO_MAX_PLANES]; + WICBitmapPlane planes[GST_VIDEO_MAX_PLANES]; + HRESULT hr; + GstVideoFrame frame; + guint num_planes = GST_VIDEO_INFO_N_PLANES (&self->info); + + for (guint i = 0; i < num_planes; i++) { + hr = factory->CreateBitmap (self->plane_desc[i].Width, + self->plane_desc[i].Height, + self->plane_desc[i].Format, WICBitmapCacheOnLoad, &bitmaps[i]); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create bitmap, hr: 0x%x", (guint) hr); + return FALSE; + } + + hr = gst_wic_lock_bitmap (bitmaps[i].Get (), nullptr, + WICBitmapLockRead | WICBitmapLockWrite, &locks[i], &planes[i]); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to lock bitmap for plane %d", i); + return FALSE; + } + } + + hr = transform->CopyPixels (nullptr, self->info.width, self->info.height, + WICBitmapTransformRotate0, WICPlanarOptionsPreserveSubsampling, planes, + num_planes); + + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to copy pixels, hr: 0x%x", (guint) hr); + return FALSE; + } + + if (!gst_video_frame_map (&frame, &self->info, buffer, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Failed to map output buffer"); + return FALSE; + } + + for (guint i = 0; i < num_planes; i++) { + WICBitmapPlane *plane = &planes[i]; + guint height, width_in_bytes; + guint8 *src, *dst; + guint src_stride, dst_stride; + + src = plane->pbBuffer; + dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, i); + + src_stride = plane->cbStride; + dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, i); + + width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&frame, i) * + GST_VIDEO_FRAME_COMP_PSTRIDE (&frame, i); + height = GST_VIDEO_FRAME_COMP_HEIGHT (&frame, i); + + for (guint j = 0; j < height; j++) { + memcpy (dst, src, width_in_bytes); + src += src_stride; + dst += dst_stride; + } + } + + gst_video_frame_unmap (&frame); + + return TRUE; +} + +static gboolean +gst_wic_jpeg_dec_fill_rgb_output (GstWicJpegDec * self, + IWICImagingFactory * factory, IWICBitmapSource * source, GstBuffer * buffer) +{ + ComPtr < IWICBitmap > bitmap; + ComPtr < IWICBitmapLock > bitmap_lock; + WICBitmapPlane plane; + HRESULT hr; + GstVideoFrame frame; + guint8 *src, *dst; + guint src_stride, dst_stride; + guint height, width_in_bytes; + + hr = factory->CreateBitmapFromSource (source, WICBitmapCacheOnDemand, + &bitmap); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create bitmap from source, hr: 0x%x", + (guint) hr); + return FALSE; + } + + hr = gst_wic_lock_bitmap (bitmap.Get (), nullptr, + WICBitmapLockRead, &bitmap_lock, &plane); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to lock bitmap"); + return FALSE; + } + + if (!gst_video_frame_map (&frame, &self->info, buffer, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Failed to map output buffer"); + return FALSE; + } + + src = plane.pbBuffer; + dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + + src_stride = plane.cbStride; + dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, 0); + + width_in_bytes = + GST_VIDEO_FRAME_COMP_WIDTH (&frame, 0) * + GST_VIDEO_FRAME_COMP_PSTRIDE (&frame, 0); + height = GST_VIDEO_FRAME_COMP_HEIGHT (&frame, 0); + + for (guint i = 0; i < height; i++) { + memcpy (dst, src, width_in_bytes); + src += src_stride; + dst += dst_stride; + } + + gst_video_frame_unmap (&frame); + + return TRUE; +} + +static void +gst_wic_jpeg_dec_update_output_state (GstWicJpegDec * self) +{ + GstWicDecoder *wic = GST_WIC_DECODER (self); + GstVideoDecoder *vdec = GST_VIDEO_DECODER (self); + GstVideoCodecState *output_state; + GstVideoInfo *info = &self->info; + GstVideoInfo *output_info; + + output_state = gst_video_decoder_get_output_state (vdec); + if (output_state) { + output_info = &output_state->info; + if (GST_VIDEO_INFO_FORMAT (output_info) == GST_VIDEO_INFO_FORMAT (info) && + GST_VIDEO_INFO_WIDTH (output_info) == GST_VIDEO_INFO_WIDTH (info) && + GST_VIDEO_INFO_HEIGHT (output_info) == GST_VIDEO_INFO_HEIGHT (info)) { + gst_video_codec_state_unref (output_state); + return; + } + gst_video_codec_state_unref (output_state); + } + + output_state = gst_video_decoder_set_output_state (vdec, + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info), + GST_VIDEO_INFO_HEIGHT (info), wic->input_state); + + /* Update colorimetry and chroma-site if upstream is not specified */ + if (GST_VIDEO_INFO_IS_YUV (info)) { + GstStructure *s = gst_caps_get_structure (wic->input_state->caps, 0); + + if (wic->input_state->info.chroma_site == GST_VIDEO_CHROMA_SITE_UNKNOWN) + output_state->info.chroma_site = GST_VIDEO_CHROMA_SITE_NONE; + + if (!gst_structure_get_string (s, "colorimetry")) { + output_state->info.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255; + output_state->info.colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT601; + output_state->info.colorimetry.transfer = GST_VIDEO_TRANSFER_UNKNOWN; + output_state->info.colorimetry.primaries = + GST_VIDEO_COLOR_PRIMARIES_UNKNOWN; + } + } + + gst_video_codec_state_unref (output_state); + gst_video_decoder_negotiate (vdec); + + return; +} + +static gboolean +gst_wic_jpeg_dec_set_format (GstWicDecoder * decoder, + GstVideoCodecState * state) +{ + GstWicJpegDec *self = GST_WIC_JPEG_DEC (decoder); + + gst_video_info_init (&self->info); + + return TRUE; +} + +static GstFlowReturn +gst_wic_jpeg_dec_process_output (GstWicDecoder * decoder, + IWICImagingFactory * factory, IWICBitmapFrameDecode * decode_frame, + GstVideoCodecFrame * frame) +{ + GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder); + GstWicJpegDec *self = GST_WIC_JPEG_DEC (decoder); + GstVideoInfo *info = &decoder->input_state->info; + ComPtr < IWICBitmapSource > source; + ComPtr < IWICBitmapSource > input; + ComPtr < IWICPlanarBitmapSourceTransform > transform; + ComPtr < IWICJpegFrameDecode > jpeg_decode; + UINT width, height; + HRESULT hr; + guint out_width, out_height; + GstFlowReturn flow_ret; + gboolean rst; + WICJpegFrameHeader hdr; + + hr = decode_frame->GetSize (&width, &height); + if (FAILED (hr)) { + GST_ERROR_OBJECT (decoder, "Failed to get size, hr: 0x%x", (guint) hr); + goto error; + } + + hr = decode_frame->QueryInterface (IID_PPV_ARGS (&jpeg_decode)); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "IWICJpegFrameDecode interface is not supported"); + goto error; + } + + hr = jpeg_decode->GetFrameHeader (&hdr); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to get frame header, hr:0x%x", (guint) hr); + goto error; + } + + /* JPEG may have interlaced stream, but WIC supports only single jpeg + * frame per run (other field may be dropped), configure scaler to + * workaround it */ + if (width == info->width && 2 * height == info->height) { + ComPtr < IWICBitmapScaler > scaler; + + GST_LOG_OBJECT (decoder, + "Need scale %dx%d -> %dx%d", width, height, info->width, info->height); + out_width = info->width; + out_height = info->height; + + hr = factory->CreateBitmapScaler (&scaler); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create scaler, hr: 0x%x", (guint) hr); + goto error; + } + + hr = scaler->Initialize (decode_frame, out_width, out_height, + WICBitmapInterpolationModeHighQualityCubic); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Unable to initialize scaler, hr: 0x%x", + (guint) hr); + goto error; + } + + scaler.As (&input); + } else { + out_width = width; + out_height = height; + input = decode_frame; + } + + /* WIC JPEG decoder supports GRAY8, BGR and CMYK pixel formats as native + * output formats, and staring with Windows 8.1, YUV formats support was added. + * See also + * https://docs.microsoft.com/en-us/windows/win32/wic/-wic-codec-native-pixel-formats#jpeg-native-codec + * https://docs.microsoft.com/en-us/windows/win32/wic/jpeg-ycbcr-support + * + * This element will output the native pixel format if possible, but + * conversion will/should happen for 4:4:0 YUV or CMYK since we don't have any + * defined format for those formats + */ + rst = gst_wic_jpeg_dec_prepare_yuv_output (self, + &hdr, input.Get (), out_width, out_height, &transform); + + if (!rst) { + rst = gst_wic_jpeg_dec_prepare_rgb_output (self, factory, input.Get (), + out_width, out_height, &source); + } + + if (!rst) + goto error; + + gst_wic_jpeg_dec_update_output_state (self); + + flow_ret = gst_video_decoder_allocate_output_frame (vdec, frame); + if (flow_ret != GST_FLOW_OK) { + gst_video_decoder_release_frame (vdec, frame); + GST_INFO_OBJECT (self, "Unable to allocate output"); + return flow_ret; + } + + if (transform) { + rst = gst_wic_jpeg_dec_fill_yuv_output (self, factory, transform.Get (), + frame->output_buffer); + } else { + rst = gst_wic_jpeg_dec_fill_rgb_output (self, factory, source.Get (), + frame->output_buffer); + } + + if (!rst) + goto error; + + return gst_video_decoder_finish_frame (vdec, frame); + +error: + gst_video_decoder_release_frame (vdec, frame); + return GST_FLOW_ERROR; +} diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.h b/subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.h new file mode 100644 index 0000000..8babb3a --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicjpegdec.h @@ -0,0 +1,31 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include "gstwicdecoder.h" + +G_BEGIN_DECLS + +#define GST_TYPE_WIC_JPEG_DEC (gst_wic_jpeg_dec_get_type()) +G_DECLARE_FINAL_TYPE (GstWicJpegDec, gst_wic_jpeg_dec, + GST, WIC_JPEG_DEC, GstWicDecoder); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.cpp b/subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.cpp new file mode 100644 index 0000000..07093a0 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.cpp @@ -0,0 +1,315 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-wicpngdec + * @title: wicpngdec + * + * This element decodes PNG compressed data into RAW video data. + * + * Since: 1.22 + * + */ + +#include "gstwicpngdec.h" + +#include +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +GST_DEBUG_CATEGORY_STATIC (gst_wic_png_dec_debug); +#define GST_CAT_DEFAULT gst_wic_png_dec_debug + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("image/png") + ); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE + ("{ RGBA64_LE, BGRA, RGBA, BGR, RGB, GRAY8, GRAY16_BE }")) + ); + + +struct _GstWicPngDec +{ + GstWicDecoder parent; + + WICBitmapPlaneDescription plane_desc[GST_VIDEO_MAX_PLANES]; + + GstVideoInfo info; +}; + +static gboolean gst_wic_png_dec_set_format (GstWicDecoder * decoder, + GstVideoCodecState * state); +static GstFlowReturn gst_wic_png_dec_process_output (GstWicDecoder * decoder, + IWICImagingFactory * factory, IWICBitmapFrameDecode * decode_frame, + GstVideoCodecFrame * frame); + +#define gst_wic_decoder_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstWicPngDec, gst_wic_png_dec, GST_TYPE_WIC_DECODER, + GST_DEBUG_CATEGORY_INIT (gst_wic_png_dec_debug, + "wicpngdec", 0, "wicpngdec")); + +static void +gst_wic_png_dec_class_init (GstWicPngDecClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstWicDecoderClass *decoder_class = GST_WIC_DECODER_CLASS (klass); + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + + gst_element_class_set_static_metadata (element_class, + "Windows Imaging Component PNG decoder", "Codec/Decoder/Image", + "Png image decoder using Windows Imaging Component API", + "Seungha Yang "); + + decoder_class->codec_id = GUID_ContainerFormatPng; + decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_wic_png_dec_set_format); + decoder_class->process_output = + GST_DEBUG_FUNCPTR (gst_wic_png_dec_process_output); +} + +static void +gst_wic_png_dec_init (GstWicPngDec * self) +{ + gst_video_info_init (&self->info); + gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), TRUE); +} + +static gboolean +gst_wic_png_dec_prepare_output (GstWicPngDec * self, + IWICImagingFactory * factory, IWICBitmapSource * input, + guint out_width, guint out_height, IWICBitmapSource ** source) +{ + WICPixelFormatGUID native_pixel_format; + GstVideoFormat native_format = GST_VIDEO_FORMAT_UNKNOWN; + HRESULT hr; + ComPtr < IWICBitmapSource > output; + + hr = input->GetPixelFormat (&native_pixel_format); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to query pixel format, hr: 0x%x", + (guint) hr); + return FALSE; + } + + if (!gst_wic_pixel_format_to_gst (native_pixel_format, &native_format)) { + ComPtr < IWICFormatConverter > conv; + + GST_LOG_OBJECT (self, + "Native format is not supported for output, needs conversion"); + + native_format = GST_VIDEO_FORMAT_BGRA; + if (IsEqualGUID (native_pixel_format, GUID_WICPixelFormat1bppIndexed) + || IsEqualGUID (native_pixel_format, GUID_WICPixelFormat2bppIndexed) + || IsEqualGUID (native_pixel_format, GUID_WICPixelFormat4bppIndexed) + || IsEqualGUID (native_pixel_format, GUID_WICPixelFormat8bppIndexed)) { + /* palette, convert to BGRA */ + native_format = GST_VIDEO_FORMAT_BGRA; + } else if (IsEqualGUID (native_pixel_format, GUID_WICPixelFormatBlackWhite) + || IsEqualGUID (native_pixel_format, GUID_WICPixelFormat2bppGray) + || IsEqualGUID (native_pixel_format, GUID_WICPixelFormat4bppGray)) { + /* gray scale */ + native_format = GST_VIDEO_FORMAT_GRAY8; + } else if (IsEqualGUID (native_pixel_format, GUID_WICPixelFormat48bppRGB)) { + /* 16bits per channel RGB, do we have defined format? */ + native_format = GST_VIDEO_FORMAT_RGBA64_LE; + } + + if (!gst_wic_pixel_format_from_gst (native_format, &native_pixel_format)) { + GST_ERROR_OBJECT (self, "Failed to convert format to WIC"); + return FALSE; + } + + hr = factory->CreateFormatConverter (&conv); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create converter"); + return FALSE; + } + + hr = conv->Initialize (input, native_pixel_format, + WICBitmapDitherTypeNone, nullptr, 0.0f, WICBitmapPaletteTypeCustom); + + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to initialize converter, hr: 0x%x", + (guint) hr); + return FALSE; + } + + conv.As (&output); + } else { + output = input; + } + + gst_video_info_set_format (&self->info, native_format, out_width, out_height); + + *source = output.Detach (); + + return TRUE; +} + +static gboolean +gst_wic_png_dec_fill_output (GstWicPngDec * self, + IWICImagingFactory * factory, IWICBitmapSource * source, GstBuffer * buffer) +{ + ComPtr < IWICBitmap > bitmap; + ComPtr < IWICBitmapLock > bitmap_lock; + WICBitmapPlane plane; + HRESULT hr; + GstVideoFrame frame; + guint8 *src, *dst; + guint src_stride, dst_stride; + guint height, width_in_bytes; + + hr = factory->CreateBitmapFromSource (source, WICBitmapCacheOnDemand, + &bitmap); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to create bitmap from source, hr: 0x%x", + (guint) hr); + return FALSE; + } + + hr = gst_wic_lock_bitmap (bitmap.Get (), nullptr, + WICBitmapLockRead, &bitmap_lock, &plane); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Failed to lock bitmap"); + return FALSE; + } + + if (!gst_video_frame_map (&frame, &self->info, buffer, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Failed to map output buffer"); + return FALSE; + } + + src = plane.pbBuffer; + dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + + src_stride = plane.cbStride; + dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, 0); + + width_in_bytes = + GST_VIDEO_FRAME_COMP_WIDTH (&frame, 0) * + GST_VIDEO_FRAME_COMP_PSTRIDE (&frame, 0); + height = GST_VIDEO_FRAME_COMP_HEIGHT (&frame, 0); + + for (guint i = 0; i < height; i++) { + memcpy (dst, src, width_in_bytes); + src += src_stride; + dst += dst_stride; + } + + gst_video_frame_unmap (&frame); + + return TRUE; +} + +static void +gst_wic_png_dec_update_output_state (GstWicPngDec * self) +{ + GstWicDecoder *wic = GST_WIC_DECODER (self); + GstVideoDecoder *vdec = GST_VIDEO_DECODER (self); + GstVideoCodecState *output_state; + GstVideoInfo *info = &self->info; + GstVideoInfo *output_info; + + output_state = gst_video_decoder_get_output_state (vdec); + if (output_state) { + output_info = &output_state->info; + if (GST_VIDEO_INFO_FORMAT (output_info) == GST_VIDEO_INFO_FORMAT (info) && + GST_VIDEO_INFO_WIDTH (output_info) == GST_VIDEO_INFO_WIDTH (info) && + GST_VIDEO_INFO_HEIGHT (output_info) == GST_VIDEO_INFO_HEIGHT (info)) { + gst_video_codec_state_unref (output_state); + return; + } + gst_video_codec_state_unref (output_state); + } + + output_state = gst_video_decoder_set_output_state (vdec, + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info), + GST_VIDEO_INFO_HEIGHT (info), wic->input_state); + + gst_video_codec_state_unref (output_state); + gst_video_decoder_negotiate (vdec); + + return; +} + +static gboolean +gst_wic_png_dec_set_format (GstWicDecoder * decoder, GstVideoCodecState * state) +{ + GstWicPngDec *self = GST_WIC_PNG_DEC (decoder); + + gst_video_info_init (&self->info); + + return TRUE; +} + +static GstFlowReturn +gst_wic_png_dec_process_output (GstWicDecoder * decoder, + IWICImagingFactory * factory, IWICBitmapFrameDecode * decode_frame, + GstVideoCodecFrame * frame) +{ + GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder); + GstWicPngDec *self = GST_WIC_PNG_DEC (decoder); + ComPtr < IWICBitmapSource > source; + ComPtr < IWICPlanarBitmapSourceTransform > transform; + UINT width, height; + HRESULT hr; + GstFlowReturn flow_ret; + gboolean rst; + + hr = decode_frame->GetSize (&width, &height); + if (FAILED (hr)) { + GST_ERROR_OBJECT (decoder, "Failed to get size, hr: 0x%x", (guint) hr); + goto error; + } + + rst = gst_wic_png_dec_prepare_output (self, factory, decode_frame, + width, height, &source); + if (!rst) + goto error; + + gst_wic_png_dec_update_output_state (self); + + flow_ret = gst_video_decoder_allocate_output_frame (vdec, frame); + if (flow_ret != GST_FLOW_OK) { + gst_video_decoder_release_frame (vdec, frame); + GST_INFO_OBJECT (self, "Unable to allocate output"); + return flow_ret; + } + + rst = gst_wic_png_dec_fill_output (self, factory, source.Get (), + frame->output_buffer); + if (!rst) + goto error; + + return gst_video_decoder_finish_frame (vdec, frame); + +error: + gst_video_decoder_release_frame (vdec, frame); + return GST_FLOW_ERROR; +} diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.h b/subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.h new file mode 100644 index 0000000..7dccdd8 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicpngdec.h @@ -0,0 +1,31 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include "gstwicdecoder.h" + +G_BEGIN_DECLS + +#define GST_TYPE_WIC_PNG_DEC (gst_wic_png_dec_get_type()) +G_DECLARE_FINAL_TYPE (GstWicPngDec, gst_wic_png_dec, + GST, WIC_PNG_DEC, GstWicDecoder); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicutils.cpp b/subprojects/gst-plugins-bad/sys/wic/gstwicutils.cpp new file mode 100644 index 0000000..aa0cd86 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicutils.cpp @@ -0,0 +1,123 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstwicutils.h" + +#include + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +GST_DEBUG_CATEGORY_EXTERN (gst_wic_utils_debug); +#define GST_CAT_DEFAULT gst_wic_utils_debug + +static struct +{ + WICPixelFormatGUID guid; + GstVideoFormat format; +} format_map[] = { + /* GRAY */ + {GUID_WICPixelFormat8bppGray, GST_VIDEO_FORMAT_GRAY8}, + {GUID_WICPixelFormat16bppGray, GST_VIDEO_FORMAT_GRAY16_LE}, + + /* RGB/BGR */ + {GUID_WICPixelFormat24bppRGB, GST_VIDEO_FORMAT_RGB}, + {GUID_WICPixelFormat24bppBGR, GST_VIDEO_FORMAT_BGR}, + {GUID_WICPixelFormat32bppRGB, GST_VIDEO_FORMAT_RGBx}, + {GUID_WICPixelFormat32bppBGR, GST_VIDEO_FORMAT_BGRx}, + {GUID_WICPixelFormat32bppRGBA, GST_VIDEO_FORMAT_RGBA}, + {GUID_WICPixelFormat32bppBGRA, GST_VIDEO_FORMAT_BGRA}, + {GUID_WICPixelFormat64bppRGBA, GST_VIDEO_FORMAT_RGBA64_LE}, +}; + +gboolean +gst_wic_pixel_format_to_gst (REFWICPixelFormatGUID guid, + GstVideoFormat * format) +{ + g_return_val_if_fail (format != nullptr, FALSE); + + *format = GST_VIDEO_FORMAT_UNKNOWN; + + for (guint i = 0; i < G_N_ELEMENTS (format_map); i++) { + if (IsEqualGUID (format_map[i].guid, guid)) { + *format = format_map[i].format; + return TRUE; + } + } + + return FALSE; +} + +gboolean +gst_wic_pixel_format_from_gst (GstVideoFormat format, WICPixelFormatGUID * guid) +{ + g_return_val_if_fail (guid != nullptr, FALSE); + + *guid = GUID_WICPixelFormatUndefined; + + for (guint i = 0; i < G_N_ELEMENTS (format_map); i++) { + if (format_map[i].format == format) { + *guid = format_map[i].guid; + return TRUE; + } + } + + return FALSE; +} + +HRESULT +gst_wic_lock_bitmap (IWICBitmap * bitmap, const WICRect * rect, + DWORD lock_flags, IWICBitmapLock ** bitmap_lock, WICBitmapPlane * plane) +{ + ComPtr < IWICBitmapLock > lock; + HRESULT hr; + + g_return_val_if_fail (bitmap != nullptr, E_INVALIDARG); + g_return_val_if_fail (bitmap_lock != nullptr, E_INVALIDARG); + g_return_val_if_fail (plane != nullptr, E_INVALIDARG); + + hr = bitmap->Lock (rect, lock_flags, &lock); + if (FAILED (hr)) { + GST_ERROR ("Failed to lock plane hr: 0x%x", (guint) hr); + return hr; + } + + hr = lock->GetStride (&plane->cbStride); + if (FAILED (hr)) { + GST_ERROR ("Failed get stride, hr: 0x%x", (guint) hr); + return hr; + } + + hr = lock->GetDataPointer (&plane->cbBufferSize, &plane->pbBuffer); + if (FAILED (hr)) { + GST_ERROR ("Failed to get data pointer, hr: 0x%x", (guint) hr); + return hr; + } + + hr = lock->GetPixelFormat (&plane->Format); + if (FAILED (hr)) { + GST_ERROR ("Failed to get pixel format, hr: 0x%x", (guint) hr); + return hr; + } + + *bitmap_lock = lock.Detach (); + + return S_OK; +} diff --git a/subprojects/gst-plugins-bad/sys/wic/gstwicutils.h b/subprojects/gst-plugins-bad/sys/wic/gstwicutils.h new file mode 100644 index 0000000..43fcd96 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/gstwicutils.h @@ -0,0 +1,53 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#ifndef INITGUID +#include +#endif + +#include +#include + +G_BEGIN_DECLS + +#define GST_WIC_CLEAR_COM(obj) G_STMT_START { \ + if (obj) { \ + (obj)->Release (); \ + (obj) = NULL; \ + } \ + } G_STMT_END + +gboolean gst_wic_pixel_format_to_gst (REFWICPixelFormatGUID guid, + GstVideoFormat * format); + +gboolean gst_wic_pixel_format_from_gst (GstVideoFormat format, + WICPixelFormatGUID * guid); + +HRESULT gst_wic_lock_bitmap (IWICBitmap * bitmap, + const WICRect * rect, + DWORD lock_flags, + IWICBitmapLock ** bitmap_lock, + WICBitmapPlane * plane); + +G_END_DECLS diff --git a/subprojects/gst-plugins-bad/sys/wic/meson.build b/subprojects/gst-plugins-bad/sys/wic/meson.build new file mode 100644 index 0000000..2a48e9c --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/meson.build @@ -0,0 +1,79 @@ +wic_sources = [ + 'gstwicdecoder.cpp', + 'gstwicimagingfactory.cpp', + 'gstwicjpegdec.cpp', + 'gstwicpngdec.cpp', + 'gstwicutils.cpp', + 'plugin.cpp', +] + +extra_args = [] +wic_deps = [] + +wic_option = get_option('wic') +if host_system != 'windows' or mf_option.disabled() + subdir_done() +endif + +if cc.get_id() != 'msvc' + if wic_option.enabled() + error('wic plugin can only be built with MSVC') + endif + subdir_done() +endif + +windowscodecs_lib = cc.find_library('windowscodecs', required : mf_option) +have_wic = windowscodecs_lib.found() and cc.has_header('wincodec.h') and cc.has_header('wincodecsdk.h') +if not have_wic + if wic_option.enabled() + error('The wic plugin was enabled explicitly, but required libraries were not found.') + endif + subdir_done() +endif + +wic_deps += [windowscodecs_lib] + +win10_sdk = cxx.compiles('''#include + #ifndef WDK_NTDDI_VERSION + #error "unknown Windows SDK version" + #endif + #if (WDK_NTDDI_VERSION < 0x0A000000) + #error "Not a Windows 10 SDK" + #endif + ''', + name: 'building with Windows 10 SDK') + +if not win10_sdk + if wic_option.enabled() + error('wic plugin was enabled explicitly, but Windows 10 SDK is unavailable') + else + subdir_done() + endif +endif + +building_for_win10 = cxx.compiles('''#include + #ifndef WINVER + #error "unknown minimum supported OS version" + #endif + #if (WINVER < 0x0A00) + #error "Windows 10 API is not guaranteed" + #endif + ''', + name: 'building for Windows 10') + +if not building_for_win10 + message('Bumping target Windows version to Windows 10 for building wic plugin') + extra_args += ['-DWINVER=0x0A00', '-D_WIN32_WINNT=0x0A00', '-DNTDDI_VERSION=WDK_NTDDI_VERSION'] +endif + +gstwic = library('gstwic', + wic_sources, + c_args : gst_plugins_bad_args + extra_args, + cpp_args : gst_plugins_bad_args + extra_args, + include_directories : [configinc], + dependencies : [gstbase_dep, gstvideo_dep] + wic_deps, + install : true, + install_dir : plugins_install_dir, +) +pkgconfig.generate(gstwic, install_dir : plugins_pkgconfig_install_dir) +plugins += [gstwic] diff --git a/subprojects/gst-plugins-bad/sys/wic/plugin.cpp b/subprojects/gst-plugins-bad/sys/wic/plugin.cpp new file mode 100644 index 0000000..80ac55b --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/wic/plugin.cpp @@ -0,0 +1,77 @@ +/* GStreamer + * Copyright (C) 2022 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "gstwicimagingfactory.h" +#include "gstwicjpegdec.h" +#include "gstwicpngdec.h" + +GST_DEBUG_CATEGORY (gst_wic_debug); +GST_DEBUG_CATEGORY (gst_wic_utils_debug); + +#define GST_CAT_DEFAULT gst_wic_debug + +static gboolean +plugin_init (GstPlugin * plugin) +{ + GstWicImagingFactory *factory; + HRESULT hr; + + GST_DEBUG_CATEGORY_INIT (gst_wic_debug, + "wic", 0, "Windows Imaging Component"); + GST_DEBUG_CATEGORY_INIT (gst_wic_utils_debug, "wicutils", 0, "wicutils"); + + factory = gst_wic_imaging_factory_new (); + if (!factory) + return TRUE; + + hr = gst_wic_imaging_factory_check_codec_support (factory, + TRUE, GUID_ContainerFormatJpeg); + if (SUCCEEDED (hr)) { + gst_element_register (plugin, + "wicjpegdec", GST_RANK_SECONDARY, GST_TYPE_WIC_JPEG_DEC); + } else { + GST_INFO_OBJECT (factory, + "Jpeg decoder is not supported, hr: 0x%x", (guint) hr); + } + + hr = gst_wic_imaging_factory_check_codec_support (factory, + TRUE, GUID_ContainerFormatPng); + if (SUCCEEDED (hr)) { + gst_element_register (plugin, + "wicpngdec", GST_RANK_SECONDARY, GST_TYPE_WIC_PNG_DEC); + } else { + GST_INFO_OBJECT (factory, + "Png decoder is not supported, hr: 0x%x", (guint) hr); + } + + gst_object_unref (factory); + + return TRUE; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + wic, + "Windows Imaging Component (WIC) plugin", + plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) -- 2.7.4