From da86cec40e375d4196b5ea73b7421075b6d2ad98 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Mon, 3 Oct 2016 02:34:50 +1100 Subject: [PATCH] rpicamsrc: First attempt at implementing MJPEG and raw video support --- sys/rpicamsrc/RaspiCapture.c | 169 ++++++++++++++++++++++++++++--------------- sys/rpicamsrc/RaspiCapture.h | 5 ++ sys/rpicamsrc/gstrpicamsrc.c | 70 ++++++++++++++---- 3 files changed, 170 insertions(+), 74 deletions(-) diff --git a/sys/rpicamsrc/RaspiCapture.c b/sys/rpicamsrc/RaspiCapture.c index abe5346..ad4798f 100644 --- a/sys/rpicamsrc/RaspiCapture.c +++ b/sys/rpicamsrc/RaspiCapture.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2015 Jan Schmidt + * Copyright (c) 2013-2016 Jan Schmidt Portions: Copyright (c) 2013, Broadcom Europe Ltd Copyright (c) 2013, James Hughes @@ -269,6 +269,7 @@ void raspicapture_default_config(RASPIVID_CONFIG *config) config->demoInterval = 250; // ms config->immutableInput = 1; config->profile = MMAL_VIDEO_PROFILE_H264_HIGH; + config->encoding = MMAL_ENCODING_H264; config->bInlineHeaders = 0; @@ -1119,9 +1120,6 @@ raspi_capture_set_format_and_start(RASPIVID_STATE *state) format = preview_port->format; - format->encoding = MMAL_ENCODING_OPAQUE; - format->encoding_variant = MMAL_ENCODING_I420; - if(config->camera_parameters.shutter_speed > 6000000) { MMAL_PARAMETER_FPS_RANGE_T fps_range = {{MMAL_PARAMETER_FPS_RANGE, sizeof(fps_range)}, @@ -1147,6 +1145,8 @@ raspi_capture_set_format_and_start(RASPIVID_STATE *state) } format->encoding = MMAL_ENCODING_OPAQUE; + format->encoding_variant = MMAL_ENCODING_I420; + format->es->video.width = VCOS_ALIGN_UP(config->width, 32); format->es->video.height = VCOS_ALIGN_UP(config->height, 16); format->es->video.crop.x = 0; @@ -1165,9 +1165,7 @@ raspi_capture_set_format_and_start(RASPIVID_STATE *state) } // Set the encode format on the video port - format = video_port->format; - format->encoding_variant = MMAL_ENCODING_I420; if(config->camera_parameters.shutter_speed > 6000000) { @@ -1182,7 +1180,16 @@ raspi_capture_set_format_and_start(RASPIVID_STATE *state) mmal_port_parameter_set(video_port, &fps_range.hdr); } - format->encoding = MMAL_ENCODING_OPAQUE; + /* If encoding, set opaque tunneling format */ + if (state->encoder_component) { + format->encoding = MMAL_ENCODING_OPAQUE; + format->encoding_variant = MMAL_ENCODING_I420; + } + else { + format->encoding = config->encoding; + format->encoding_variant = config->encoding; + } + format->es->video.width = VCOS_ALIGN_UP(config->width, 32); format->es->video.height = VCOS_ALIGN_UP(config->height, 16); format->es->video.crop.x = 0; @@ -1279,6 +1286,10 @@ gboolean raspi_capture_request_i_frame(RASPIVID_STATE *state) MMAL_PORT_T *encoder_output = NULL; MMAL_STATUS_T status; MMAL_PARAMETER_BOOLEAN_T param = {{ MMAL_PARAMETER_VIDEO_REQUEST_I_FRAME, sizeof(param)}, 1}; + + if (state->encoder_component) + return TRUE; + encoder_output = state->encoder_component->output[0]; status = mmal_port_parameter_set(encoder_output, ¶m.hdr); if (status != MMAL_SUCCESS) @@ -1302,15 +1313,24 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) MMAL_COMPONENT_T *encoder = 0; MMAL_PORT_T *encoder_input = NULL, *encoder_output = NULL; MMAL_STATUS_T status; - MMAL_POOL_T *pool; RASPIVID_CONFIG *config = &state->config; - status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_ENCODER, &encoder); + gboolean encoded_format = + (config->encoding == MMAL_ENCODING_H264 || + config->encoding == MMAL_ENCODING_MJPEG || + config->encoding == MMAL_ENCODING_JPEG); - if (status != MMAL_SUCCESS) - { - vcos_log_error("Unable to create video encoder component"); - goto error; + if (!encoded_format) + return MMAL_SUCCESS; + + if (config->encoding == MMAL_ENCODING_JPEG) + status = mmal_component_create(MMAL_COMPONENT_DEFAULT_IMAGE_ENCODER, &encoder); + else + status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_ENCODER, &encoder); + + if (status != MMAL_SUCCESS) { + vcos_log_error("Unable to create video encoder component"); + goto error; } if (!encoder->input_num || !encoder->output_num) @@ -1326,23 +1346,27 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) // We want same format on input and output mmal_format_copy(encoder_output->format, encoder_input->format); - // Only supporting H264 at the moment - encoder_output->format->encoding = MMAL_ENCODING_H264; + // Configure desired encoding + encoder_output->format->encoding = config->encoding; encoder_output->format->bitrate = config->bitrate; - encoder_output->buffer_size = encoder_output->buffer_size_recommended; + if (config->encoding == MMAL_ENCODING_H264) + encoder_output->buffer_size = encoder_output->buffer_size_recommended; + else + encoder_output->buffer_size = 256<<10; if (encoder_output->buffer_size < encoder_output->buffer_size_min) encoder_output->buffer_size = encoder_output->buffer_size_min; - GST_DEBUG ("encoder buffer size is %u", (guint)encoder_output->buffer_size); - encoder_output->buffer_num = encoder_output->buffer_num_recommended; if (encoder_output->buffer_num < encoder_output->buffer_num_min) encoder_output->buffer_num = encoder_output->buffer_num_min; + GST_DEBUG ("encoder wants %d buffers of size %u", + (guint)encoder_output->buffer_num, (guint)encoder_output->buffer_size); + // We need to set the frame rate on output to 0, to ensure it gets // updated correctly from the input framerate when port connected encoder_output->format->es->video.frame_rate.num = 0; @@ -1350,9 +1374,7 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) // Commit the port changes to the output port status = mmal_port_format_commit(encoder_output); - - if (status != MMAL_SUCCESS) - { + if (status != MMAL_SUCCESS) { vcos_log_error("Unable to set format on video encoder output port"); goto error; } @@ -1370,7 +1392,7 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) } - if (config->intraperiod != -1) + if (config->encoding == MMAL_ENCODING_H264 && config->intraperiod != -1) { MMAL_PARAMETER_UINT32_T param = {{ MMAL_PARAMETER_INTRAPERIOD, sizeof(param)}, config->intraperiod}; status = mmal_port_parameter_set(encoder_output, ¶m.hdr); @@ -1381,7 +1403,7 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) } } - if (config->quantisationParameter) + if (config->encoding == MMAL_ENCODING_H264 && config->quantisationParameter) { MMAL_PARAMETER_UINT32_T param = {{ MMAL_PARAMETER_VIDEO_ENCODE_INITIAL_QUANT, sizeof(param)}, config->quantisationParameter}; status = mmal_port_parameter_set(encoder_output, ¶m.hdr); @@ -1409,6 +1431,7 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) } + if (config->encoding == MMAL_ENCODING_H264) { MMAL_PARAMETER_VIDEO_PROFILE_T param; param.hdr.id = MMAL_PARAMETER_PROFILE; @@ -1440,14 +1463,16 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) } //set INLINE VECTORS flag to request motion vector estimates - if (mmal_port_parameter_set_boolean(encoder_output, MMAL_PARAMETER_VIDEO_ENCODE_INLINE_VECTORS, config->inlineMotionVectors) != MMAL_SUCCESS) + if (config->encoding == MMAL_ENCODING_H264 && + mmal_port_parameter_set_boolean(encoder_output, MMAL_PARAMETER_VIDEO_ENCODE_INLINE_VECTORS, config->inlineMotionVectors) != MMAL_SUCCESS) { vcos_log_error("failed to set INLINE VECTORS parameters"); // Continue rather than abort.. } // Adaptive intra refresh settings - if (config->intra_refresh_type != -1) + if (config->encoding == MMAL_ENCODING_H264 && + config->intra_refresh_type != -1) { MMAL_PARAMETER_VIDEO_INTRA_REFRESH_T param; @@ -1476,6 +1501,23 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) } } + if (config->encoding == MMAL_ENCODING_JPEG) + { + status = mmal_port_parameter_set_uint32(encoder_output, MMAL_PARAMETER_JPEG_Q_FACTOR, config->jpegQuality); + if (status != MMAL_SUCCESS) { + vcos_log_error("Unable to set JPEG quality"); + goto error; + } + +#ifdef MMAL_PARAMETER_JPEG_RESTART_INTERVAL + status = mmal_port_parameter_set_uint32(encoder_output, MMAL_PARAMETER_JPEG_RESTART_INTERVAL, config->jpegRestartInterval); + if (status != MMAL_SUCCESS) { + vcos_log_error("Unable to set JPEG restart interval"); + goto error; + } +#endif + } + // Enable component status = mmal_component_enable(encoder); @@ -1485,15 +1527,6 @@ static MMAL_STATUS_T create_encoder_component(RASPIVID_STATE *state) goto error; } - /* Create pool of buffer headers for the output port to consume */ - pool = mmal_port_pool_create(encoder_output, encoder_output->buffer_num, encoder_output->buffer_size); - - if (!pool) - { - vcos_log_error("Failed to create buffer header pool for encoder output port %s", encoder_output->name); - } - - state->encoder_pool = pool; state->encoder_component = encoder; if (config->verbose) @@ -1635,10 +1668,11 @@ raspi_capture_start(RASPIVID_STATE *state) MMAL_PORT_T *preview_input_port = NULL; MMAL_PORT_T *encoder_input_port = NULL; - if ((status = create_encoder_component(state)) != MMAL_SUCCESS) - { - vcos_log_error("%s: Failed to create encode component", __func__); - return FALSE; + MMAL_POOL_T *pool; + + if ((status = create_encoder_component(state)) != MMAL_SUCCESS) { + vcos_log_error("%s: Failed to create encode component", __func__); + return FALSE; } if (config->verbose) @@ -1646,21 +1680,38 @@ raspi_capture_start(RASPIVID_STATE *state) dump_state(state); } + state->camera_video_port = state->camera_component->output[MMAL_CAMERA_VIDEO_PORT]; + state->camera_still_port = state->camera_component->output[MMAL_CAMERA_CAPTURE_PORT]; + camera_preview_port = state->camera_component->output[MMAL_CAMERA_PREVIEW_PORT]; + preview_input_port = state->preview_state.preview_component->input[0]; + + if (state->encoder_component) { + encoder_input_port = state->encoder_component->input[0]; + state->encoder_output_port = state->encoder_component->output[0]; + } else { + state->encoder_output_port = state->camera_video_port; + } + if ((status = raspi_capture_set_format_and_start(state)) != MMAL_SUCCESS) { return FALSE; } + GST_DEBUG ("Creating pool of %d buffers of size %d", + state->encoder_output_port->buffer_num, state->encoder_output_port->buffer_size); + /* Create pool of buffer headers for the output port to consume */ + pool = mmal_port_pool_create(state->encoder_output_port, + state->encoder_output_port->buffer_num, state->encoder_output_port->buffer_size); + if (!pool) + { + vcos_log_error("Failed to create buffer header pool for encoder output port %s", + state->encoder_output_port->name); + return FALSE; + } + state->encoder_pool = pool; + if (state->config.verbose) fprintf(stderr, "Starting component connection stage\n"); - camera_preview_port = state->camera_component->output[MMAL_CAMERA_PREVIEW_PORT]; - preview_input_port = state->preview_state.preview_component->input[0]; - encoder_input_port = state->encoder_component->input[0]; - - state->camera_video_port = state->camera_component->output[MMAL_CAMERA_VIDEO_PORT]; - state->camera_still_port = state->camera_component->output[MMAL_CAMERA_CAPTURE_PORT]; - state->encoder_output_port = state->encoder_component->output[0]; - if (config->preview_parameters.wantPreview ) { if (config->verbose) @@ -1678,17 +1729,19 @@ raspi_capture_start(RASPIVID_STATE *state) } } - if (config->verbose) - fprintf(stderr, "Connecting camera stills port to encoder input port\n"); + if (state->encoder_component) { + if (config->verbose) + fprintf(stderr, "Connecting camera video port to encoder input port\n"); - // Now connect the camera to the encoder - status = connect_ports(state->camera_video_port, encoder_input_port, &state->encoder_connection); - if (status != MMAL_SUCCESS) - { - if (config->preview_parameters.wantPreview ) - mmal_connection_destroy(state->preview_connection); - vcos_log_error("%s: Failed to connect camera video port to encoder input", __func__); - return FALSE; + // Now connect the camera to the encoder + status = connect_ports(state->camera_video_port, encoder_input_port, &state->encoder_connection); + if (status != MMAL_SUCCESS) + { + if (config->preview_parameters.wantPreview ) + mmal_connection_destroy(state->preview_connection); + vcos_log_error("%s: Failed to connect camera video port to encoder input", __func__); + return FALSE; + } } // Set up our userdata - this is passed though to the callback where we need the information. @@ -1788,13 +1841,13 @@ raspi_capture_stop(RASPIVID_STATE *state) if (config->preview_parameters.wantPreview ) mmal_connection_destroy(state->preview_connection); - mmal_connection_destroy(state->encoder_connection); // Disable all our ports that are not handled by connections check_disable_port(state->camera_still_port); check_disable_port(state->encoder_output_port); if (state->encoder_component) { + mmal_connection_destroy(state->encoder_connection); mmal_component_disable(state->encoder_component); destroy_encoder_component(state); } @@ -1847,7 +1900,7 @@ raspi_capture_update_config (RASPIVID_STATE *state, RASPIVID_CONFIG *config, gbo if (!dynamic) return; - if (config->change_flags & PROP_CHANGE_ENCODING) { + if (state->encoder_component && config->change_flags & PROP_CHANGE_ENCODING) { /* BITRATE or QUANT or KEY Interval, intra refresh */ MMAL_COMPONENT_T *encoder = state->encoder_component; MMAL_PORT_T *encoder_output = encoder->output[0]; diff --git a/sys/rpicamsrc/RaspiCapture.h b/sys/rpicamsrc/RaspiCapture.h index 2462caa..6a79ac5 100644 --- a/sys/rpicamsrc/RaspiCapture.h +++ b/sys/rpicamsrc/RaspiCapture.h @@ -111,6 +111,11 @@ typedef struct int settings; /// Request settings from the camera int sensor_mode; /// Sensor mode. 0=auto. Check docs/forum for modes selected by other values. int intra_refresh_type; /// What intra refresh type to use. -1 to not set. + + MMAL_FOURCC_T encoding; // Which encoding to use + + int jpegQuality; + int jpegRestartInterval; } RASPIVID_CONFIG; typedef struct RASPIVID_STATE_T RASPIVID_STATE; diff --git a/sys/rpicamsrc/gstrpicamsrc.c b/sys/rpicamsrc/gstrpicamsrc.c index 879d0f2..3bf9ab6 100644 --- a/sys/rpicamsrc/gstrpicamsrc.c +++ b/sys/rpicamsrc/gstrpicamsrc.c @@ -57,8 +57,6 @@ #ifdef HAVE_CONFIG_H # include #endif - -#include #include #include "gstrpicamsrc.h" @@ -135,8 +133,9 @@ enum PROP_ANNOTATION_TEXT_BG_COLOUR, PROP_INTRA_REFRESH_TYPE, #ifdef GST_RPI_CAM_SRC_ENABLE_VIDEO_DIRECTION - PROP_VIDEO_DIRECTION + PROP_VIDEO_DIRECTION, #endif + PROP_JPEG_QUALITY }; #define CAMERA_DEFAULT 0 @@ -158,6 +157,8 @@ enum #define EXPOSURE_MODE_DEFAULT GST_RPI_CAM_SRC_EXPOSURE_MODE_AUTO #define EXPOSURE_METERING_MODE_DEFAULT GST_RPI_CAM_SRC_EXPOSURE_METERING_MODE_AVERAGE +#define DEFAULT_JPEG_QUALITY 50 + /* params->exposureMode = MMAL_PARAM_EXPOSUREMODE_AUTO; params->exposureMeterMode = MMAL_PARAM_EXPOSUREMETERINGMODE_AVERAGE; @@ -172,8 +173,7 @@ enum params->roi.w = params->roi.h = 1.0; */ -#define RAW_AND_JPEG_CAPS \ - GST_VIDEO_CAPS_MAKE (GST_VIDEO_FORMATS_ALL) ";" \ +#define JPEG_CAPS \ "image/jpeg," \ "width = " GST_VIDEO_SIZE_RANGE "," \ "height = " GST_VIDEO_SIZE_RANGE "," \ @@ -186,6 +186,8 @@ enum "stream-format = (string) byte-stream, " \ "alignment = (string) nal, " \ "profile = (string) { baseline, main, high }" +#define RAW_CAPS \ + GST_VIDEO_CAPS_MAKE ("{ I420, RGB, BGR, RGBA }") /* FIXME: Map more raw formats */ #ifdef GST_RPI_CAM_SRC_ENABLE_VIDEO_DIRECTION #define gst_rpi_cam_src_reset_custom_orientation(src) { src->orientation = GST_VIDEO_ORIENTATION_CUSTOM; } @@ -196,7 +198,7 @@ enum static GstStaticPadTemplate video_src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS ( /*RAW_AND_JPEG_CAPS "; " */ H264_CAPS) + GST_STATIC_CAPS ( H264_CAPS "; " JPEG_CAPS "; " RAW_CAPS) ); @@ -297,6 +299,10 @@ gst_rpi_cam_src_class_init (GstRpiCamSrcClass * klass) "Bitrate for encoding. 0 for VBR using quantisation-parameter", 0, BITRATE_HIGHEST, BITRATE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_JPEG_QUALITY, + g_param_spec_int ("jpeg-quality", "JPEG Quality", + "Quality setting for JPEG encode", 1, 100, DEFAULT_JPEG_QUALITY, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_KEYFRAME_INTERVAL, g_param_spec_int ("keyframe-interval", "Keyframe Interface", "Interval (in frames) between I frames. -1 = automatic, 0 = single-keyframe", @@ -804,6 +810,10 @@ gst_rpi_cam_src_set_property (GObject * object, guint prop_id, src->capture_config.bitrate = g_value_get_int (value); src->capture_config.change_flags |= PROP_CHANGE_ENCODING; break; + case PROP_JPEG_QUALITY: + src->capture_config.jpegQuality = g_value_get_int (value); + src->capture_config.change_flags |= PROP_CHANGE_ENCODING; + break; case PROP_KEYFRAME_INTERVAL: src->capture_config.intraperiod = g_value_get_int (value); src->capture_config.change_flags |= PROP_CHANGE_ENCODING; @@ -1013,6 +1023,9 @@ gst_rpi_cam_src_get_property (GObject * object, guint prop_id, case PROP_BITRATE: g_value_set_int (value, src->capture_config.bitrate); break; + case PROP_JPEG_QUALITY: + g_value_set_int (value, src->capture_config.jpegQuality); + break; case PROP_KEYFRAME_INTERVAL: g_value_set_int (value, src->capture_config.intraperiod); break; @@ -1267,16 +1280,41 @@ gst_rpi_cam_src_set_caps (GstBaseSrc * bsrc, GstCaps * caps) return FALSE; structure = gst_caps_get_structure (caps, 0); - profile_str = gst_structure_get_string (structure, "profile"); - if (profile_str) { - if (g_str_equal (profile_str, "baseline")) - src->capture_config.profile = MMAL_VIDEO_PROFILE_H264_BASELINE; - else if (g_str_equal (profile_str, "main")) - src->capture_config.profile = MMAL_VIDEO_PROFILE_H264_MAIN; - else if (g_str_equal (profile_str, "high")) - src->capture_config.profile = MMAL_VIDEO_PROFILE_H264_HIGH; - else - g_warning ("Unknown profile string in rpicamsrc caps: %s", profile_str); + if (gst_structure_has_name (structure, "video/x-h264")) { + src->capture_config.encoding = MMAL_ENCODING_H264; + profile_str = gst_structure_get_string (structure, "profile"); + if (profile_str) { + if (g_str_equal (profile_str, "baseline")) + src->capture_config.profile = MMAL_VIDEO_PROFILE_H264_BASELINE; + else if (g_str_equal (profile_str, "main")) + src->capture_config.profile = MMAL_VIDEO_PROFILE_H264_MAIN; + else if (g_str_equal (profile_str, "high")) + src->capture_config.profile = MMAL_VIDEO_PROFILE_H264_HIGH; + else + g_warning ("Unknown profile string in rpicamsrc caps: %s", profile_str); + } + } + else if (gst_structure_has_name (structure, "image/jpeg")) { + src->capture_config.encoding = MMAL_ENCODING_MJPEG; + } + else { + /* Raw caps */ + switch (GST_VIDEO_INFO_FORMAT(&info)) { + case GST_VIDEO_FORMAT_I420: + src->capture_config.encoding = MMAL_ENCODING_I420; + break; + case GST_VIDEO_FORMAT_RGB: + src->capture_config.encoding = MMAL_ENCODING_RGB24; + break; + case GST_VIDEO_FORMAT_BGR: + src->capture_config.encoding = MMAL_ENCODING_BGR24; + break; + case GST_VIDEO_FORMAT_RGBA: + src->capture_config.encoding = MMAL_ENCODING_RGBA; + break; + default: + return FALSE; + } } src->capture_config.width = info.width; -- 2.7.4