Transcoding: Change stream parameters

Goal

In the previous Transcoding: Connect player to recorder tutorial, we’ve seen how to connect a player to a recorder to perform basic transcoding. In this tutorial you will learn how to:

• Change recorded streams parameters in order to transform streams changing video resolution or audio channels

Prerequisites

• Event Management

• Stream Management

• Webcam Recorder

• Transcoding: Connect player to recorder

Walkthrough

The source code is basically the same as in the previous tutorial, so we will only focus on differences. First, we define a struct to configure and pass transcoded audio and video parameters to the right callback:

typedef struct
{
    gint video_width;
    gint video_height;
    gint video_framerate;
    gint video_bitrate;

    gint audio_channels;
    gint audio_sampling;
    gint audio_bitrate;
} OutputConfig;

We are setting those parameters in the application main entry point with any values we want to test:

    app.output.video_width = 1024;
    app.output.video_height = 720;
    app.output.video_framerate = 30;
    app.output.video_bitrate = 1200000;
    app.output.audio_bitrate = 56000;

Then the only thing left to do is configuring the recorder streams when they are requested. This is done in the same _recorder_request_stream function as in the previous tutorial:

FluStream *_recorder_request_stream(const App *app, FluStreamType stream_type)
{
    FluStreamInfo info = {0};
    info.type = stream_type;

    if (stream_type == FLU_STREAM_TYPE_VIDEO)
    {
        info.data.video.vcodec.type = FLU_STREAM_VIDEO_CODEC_H264;
        info.data.video.width = app->output.video_width;
        info.data.video.height = app->output.video_height;
        info.data.video.fps_n = app->output.video_framerate;
        info.data.video.fps_d = 1;
        info.data.video.bitrate = app->output.video_bitrate;
    }
    else if (stream_type == FLU_STREAM_TYPE_AUDIO)
    {
        info.data.audio.acodec = FLU_STREAM_AUDIO_CODEC_AAC;
        info.data.audio.channels = app->output.audio_channels;
        info.data.audio.rate = app->output.audio_sampling;
        info.data.audio.bitrate = app->output.audio_bitrate;
    }
    else
    {
        return NULL;
    }

    GError *error = NULL;
    FluStream *stream = flu_recorder_request_stream(app->recorder, &info, &error);
    if (!stream)
    {
        g_print("Error: cannot create %s stream for recorder (%s)\n",
                (stream_type == FLU_STREAM_TYPE_VIDEO) ? "video" : "audio",
                error ? error->message : "undefined error");
        if (error)
        {
            g_error_free(error);
        }
    }

    return stream;
}

We are using the FluStreamVideoInfo struct to configure transcoded video streams:

        info.data.video.vcodec.type = FLU_STREAM_VIDEO_CODEC_H264;
        info.data.video.width = app->output.video_width;
        info.data.video.height = app->output.video_height;
        info.data.video.fps_n = app->output.video_framerate;
        info.data.video.fps_d = 1;
        info.data.video.bitrate = app->output.video_bitrate;

And FluStreamAudioInfo struct to configure transcoded audio streams:

        info.data.audio.acodec = FLU_STREAM_AUDIO_CODEC_AAC;
        info.data.audio.channels = app->output.audio_channels;
        info.data.audio.rate = app->output.audio_sampling;
        info.data.audio.bitrate = app->output.audio_bitrate;

Full source code

Full source code

#include <fluendo-sdk.h>
#include <stdio.h>

/*****************************************************************************
 * Application structures
 *****************************************************************************/

typedef struct
{
    gint video_width;
    gint video_height;
    gint video_framerate;
    gint video_bitrate;

    gint audio_channels;
    gint audio_sampling;
    gint audio_bitrate;
} OutputConfig;

typedef struct
{
    FluStream *player_stream;
    FluStream *recorder_stream;
} Stream;

typedef struct
{
    gchar *input_uri;

    GMainLoop *main_loop;
    guint keyboard_source_id;

    FluPlayer *player;
    FluRecorder *recorder;
    OutputConfig output;

    /* Unlike previous fields which are initialized BEFORE threads creation
     * (see _init_app) and cleaned up AFTER joining all threads (see _clean_app),
     * nb_streams and streams pointer are initialized from the Fluendo-SDK events
     * thread (see _player_on_state_changed) and then read from potentially
     * different streaming threads (see _player_on_render). They are cleaned up
     * AFTER joining all threads like the rest of fields.
     * As nb_streams and streams pointer are only written once and then constant
     * until joining all threads, we only need a memory barrier (no need for locks).
     * We obtain it by using atomic for the streams pointer taking care of writing
     * nb_streams BEFORE setting the atomic pointer. */
    guint nb_streams;
    volatile Stream *streams;
} App;

/*****************************************************************************
 * Recorder events management
 *****************************************************************************/

static gboolean _recorder_on_request_save_mode(FluRecorder *recorder, FluRecorderEvent *event, gpointer data)
{
    FluRecorderEventRequestSaveMode *ev = (FluRecorderEventRequestSaveMode *)event;

    ev->mode = FLU_RECORDER_SAVE_MODE_FILE;
    ev->data.file.filename = "./output_transcoded.mp4";

    return TRUE;
}

static gboolean _recorder_on_state_changed(FluRecorder *recorder, FluRecorderEvent *event, gpointer data)
{
    FluRecorderEventState *ev = (FluRecorderEventState *)event;

    g_print("Recorder state changed to %s\n", flu_recorder_state_name_get(ev->state));

    return TRUE;
}

static gboolean _recorder_on_error(FluRecorder *recorder, FluRecorderEvent *event, gpointer data)
{
    FluRecorderEventError *ev = (FluRecorderEventError *)event;

    g_print("Error (recorder): %s %s\n", ev->error.error->message, ev->error.dbg);
    g_main_loop_quit((GMainLoop *)data);

    return FALSE;
}

/*****************************************************************************
 * Recorder creation/destruction
 *****************************************************************************/

FluRecorder *_recorder_create(GMainLoop *main_loop)
{
    FluMediaInfo media_info = {0};
    media_info.format = FLU_MEDIA_INFO_FORMAT_MP4;

    GError *error = NULL;
    FluRecorder *recorder = flu_recorder_new(&media_info, &error);
    if (recorder)
    {
        flu_recorder_event_listener_add(recorder, FLU_RECORDER_EVENT_REQUEST_SAVE_MODE, _recorder_on_request_save_mode, NULL);
        flu_recorder_event_listener_add(recorder, FLU_RECORDER_EVENT_STATE, _recorder_on_state_changed, NULL);
        flu_recorder_event_listener_add(recorder, FLU_RECORDER_EVENT_ERROR, _recorder_on_error, main_loop);
    }
    else
    {
        g_print("Error: cannot create recorder (%s)\n",
                error ? error->message : "undefined error");
        if (error)
        {
            g_error_free(error);
        }
    }

    return recorder;
}

void _recorder_destroy(FluRecorder *recorder)
{
    if (recorder)
    {
        flu_recorder_stop(recorder);
        flu_recorder_unref(recorder);
    }
}

/*****************************************************************************
 * Recorder functions
 *****************************************************************************/

FluStream *_recorder_request_stream(const App *app, FluStreamType stream_type)
{
    FluStreamInfo info = {0};
    info.type = stream_type;

    if (stream_type == FLU_STREAM_TYPE_VIDEO)
    {
        info.data.video.vcodec.type = FLU_STREAM_VIDEO_CODEC_H264;
        info.data.video.width = app->output.video_width;
        info.data.video.height = app->output.video_height;
        info.data.video.fps_n = app->output.video_framerate;
        info.data.video.fps_d = 1;
        info.data.video.bitrate = app->output.video_bitrate;
    }
    else if (stream_type == FLU_STREAM_TYPE_AUDIO)
    {
        info.data.audio.acodec = FLU_STREAM_AUDIO_CODEC_AAC;
        info.data.audio.channels = app->output.audio_channels;
        info.data.audio.rate = app->output.audio_sampling;
        info.data.audio.bitrate = app->output.audio_bitrate;
    }
    else
    {
        return NULL;
    }

    GError *error = NULL;
    FluStream *stream = flu_recorder_request_stream(app->recorder, &info, &error);
    if (!stream)
    {
        g_print("Error: cannot create %s stream for recorder (%s)\n",
                (stream_type == FLU_STREAM_TYPE_VIDEO) ? "video" : "audio",
                error ? error->message : "undefined error");
        if (error)
        {
            g_error_free(error);
        }
    }

    return stream;
}

gboolean _recorder_start(const App *app)
{
    GError *error = NULL;
    flu_recorder_record(app->recorder, &error);
    if (error)
    {
        g_print("Error: cannot start recorder (%s)\n", error->message);
        g_error_free(error);
        return FALSE;
    }

    return TRUE;
}

/*****************************************************************************
 * Player events management
 *****************************************************************************/

static gboolean _player_on_state_changed(FluPlayer *player, FluPlayerEvent *event, gpointer data)
{
    gboolean ret = TRUE;

    FluPlayerEventState *ev = (FluPlayerEventState *)event;
    g_print("Player state changed to %s\n", flu_player_state_name_get(ev->state));

    if (ev->state == FLU_PLAYER_STATE_PLAYING)
    {
        App *app = (App *)data;

        /* First time player goes to PLAYING state, we initialize a new recorder output stream
         * for each player stream, and we keep the relationships between corresponding streams.
         * This callback is called from the Fluendo-SDK event thread. As app->streams pointer
         * is only written once here and then read from streaming threads later, we use an atomic
         * in order to ensure a memory fence. */
        Stream *streams = g_atomic_pointer_get(&app->streams);
        if (!streams)
        {
            /* For the purpose of the tutorial, we just take into account video and audio streams,
             * but the same may be achieved with text and data streams too. */
            GList *player_streams = flu_player_video_active_streams_get(player);
            player_streams = g_list_concat(player_streams, flu_player_audio_active_streams_get(player));

            app->nb_streams = g_list_length(player_streams);
            streams = g_new0(Stream, app->nb_streams);

            Stream *stream_link = streams;
            for (GList *l = player_streams; l; l = l->next)
            {
                /* We don't need to keep any extra references to player or recorder streams. Weak
                 * references are enough as all links are only used within callbacks during lifetimes
                 * of the player and recorder (which already keep hard references to those streams). */
                stream_link->player_stream = (FluStream *)l->data;

                FluStreamType stream_type = flu_stream_type_get(stream_link->player_stream);
                stream_link->recorder_stream = _recorder_request_stream(app, stream_type);
                if (stream_link->recorder_stream)
                {
                    flu_stream_unref(stream_link->recorder_stream);
                }

                ++stream_link;
            }

            flu_stream_list_free(player_streams);
            g_atomic_pointer_set(&app->streams, streams);

            /* We are now ready for transcoding: let's start the recorder. */
            if (!_recorder_start(app))
            {
                ret = FALSE;
                g_main_loop_quit(app->main_loop);
            }
        }
    }

    return ret;
}

static gboolean _player_on_error(FluPlayer *player, FluPlayerEvent *event, gpointer data)
{
    FluPlayerEventError *ev = (FluPlayerEventError *)event;

    g_print("Error (player): %s %s\n", ev->error->message, ev->dbg);
    g_main_loop_quit((GMainLoop *)data);

    return FALSE;
}

static gboolean _player_on_eos(FluPlayer *player, FluPlayerEvent *event, gpointer data)
{
    g_print("End of source media reached => transcoding stopped automatically.\n");
    g_main_loop_quit((GMainLoop *)data);

    return TRUE;
}

static gboolean _player_on_request_render_mode(FluPlayer *player, FluPlayerEvent *event, gpointer data)
{
    FluPlayerEventRequestRenderMode *ev = (FluPlayerEventRequestRenderMode *)event;

    /* As we are just transcoding, we request:
     * - no visual rendering,
     * - no synchronization (buffers are treated as fast as they arrive),
     * - notification (to know when buffers are ready to be copied). */
    ev->render = FALSE;
    ev->synchronize = FALSE;
    ev->notify = TRUE;

    return TRUE;
}

static gboolean _player_on_render(FluPlayer *player, FluPlayerEvent *event, gpointer data)
{
    FluPlayerEventRender *ev = (FluPlayerEventRender *)event;
    FluStreamType stream_type = flu_stream_type_get(ev->stream);

    /* For the purpose of the tutorial, we just take into account video and audio streams,
     * but the same may be achieved with text and data streams too. */
    if ((stream_type == FLU_STREAM_TYPE_VIDEO) || (stream_type == FLU_STREAM_TYPE_AUDIO))
    {
        const App *app = (const App *)data;

        /* This callback is called from different streaming threads (for audio and
         * video streams). As app->streams pointer is initialized from another thread
         * (the event thread in _player_on_state_changed), we use an atomic to ensure
         * a memory fence. */
        Stream *streams = g_atomic_pointer_get(&app->streams);
        if (streams)
        {
            for (guint i = 0; i < app->nb_streams; ++i)
            {
                Stream *stream = streams + i;
                if (stream->player_stream == ev->stream)
                {
                    /* Write last read buffer from selected player stream to corresponding recorder stream. */
                    if (stream->recorder_stream)
                    {
                        /* Copy decoded buffer from player stream to corresponding recorder
                         * stream for further encoding. */
                        if (!flu_stream_last_sample_copy(ev->stream, stream->recorder_stream))
                        {
                            g_print("Warning: cannot copy %s buffer from player to recorder\n",
                                    (stream_type == FLU_STREAM_TYPE_VIDEO) ? "video" : "audio");
                        }
                    }
                    break;
                }
            }
        }
    }

    return TRUE;
}

/*****************************************************************************
 * Player creation/destruction
 *****************************************************************************/

FluPlayer *_player_create(App *app)
{
    FluPlayer *player = flu_player_new();

    flu_player_event_listener_add(player, FLU_PLAYER_EVENT_STATE, _player_on_state_changed, app);
    flu_player_event_listener_add(player, FLU_PLAYER_EVENT_ERROR, _player_on_error, app->main_loop);
    flu_player_event_listener_add(player, FLU_PLAYER_EVENT_EOS, _player_on_eos, app->main_loop);
    flu_player_event_listener_add(player, FLU_PLAYER_EVENT_REQUEST_RENDER_MODE, _player_on_request_render_mode, NULL);
    flu_player_event_listener_add(player, FLU_PLAYER_EVENT_RENDER, _player_on_render, app);

    return player;
}

void _player_destroy(FluPlayer *player)
{
    if (player)
    {
        flu_player_close(player);
        flu_player_unref(player);
    }
}

/*****************************************************************************
 * Application management functions
 *****************************************************************************/

static gboolean _handle_keyboard(GIOChannel *source, GIOCondition cond, gpointer data)
{
    static gboolean exiting = FALSE;

    gchar *str = NULL;
    if (g_io_channel_read_line(source, &str, NULL, NULL, NULL) == G_IO_STATUS_NORMAL)
    {
        if ((str[0] == 'q') && !exiting)
        {
            exiting = TRUE;
            g_print("Exiting...\n");
            g_main_loop_quit((GMainLoop *)data);
        }

        g_free(str);
    }

    return TRUE;
}

static void _clean_app(App *app)
{
    if (app->keyboard_source_id)
    {
        g_source_remove(app->keyboard_source_id);
        app->keyboard_source_id = 0;
    }

    _player_destroy(app->player);
    app->player = NULL;

    _recorder_destroy(app->recorder);
    app->recorder = NULL;

    /* We don't need memory fence anymore here as all threads have been
     * joined at destroying the player and the recorder. */
    g_free((Stream *)app->streams);
    app->streams = NULL;
    app->nb_streams = 0;

    g_main_loop_unref(app->main_loop);
    app->main_loop = NULL;

    flu_shutdown();

    g_free(app->input_uri);
    app->input_uri = NULL;
}

static gboolean _init_app(App *app, int argc, const char **argv)
{
    if (argc >= 2)
    {
        gchar *absolute_path = g_canonicalize_filename(argv[1], NULL);
        app->input_uri = g_filename_to_uri(absolute_path, NULL, NULL);
        g_free(absolute_path);
    }

    if (!app->input_uri)
    {
        g_print("Usage: %s [INPUT_FILE_PATH]\n", argv[0]);
        return FALSE;
    }

    flu_initialize();
    app->main_loop = g_main_loop_new(NULL, FALSE);

    app->recorder = _recorder_create(app->main_loop);
    if (!app->recorder)
    {
        _clean_app(app);
        return FALSE;
    }

    app->player = _player_create(app);
    if (!app->player)
    {
        _clean_app(app);
        return FALSE;
    }

    return TRUE;
}

/*****************************************************************************
 * Application main entry point
 *****************************************************************************/

int main(int argc, const char **argv)
{
    App app = {0};

    /* Initialize application. */
    if (!_init_app(&app, argc, argv))
    {
        return -1;
    }

    /* Configure transcoded output. */
    app.output.video_width = 1024;
    app.output.video_height = 720;
    app.output.video_framerate = 30;
    app.output.video_bitrate = 1200000;
    app.output.audio_bitrate = 56000;

    /* Start player. */
    flu_player_uri_open(app.player, app.input_uri);
    flu_player_play(app.player);

    /* Run main loop. */
    GIOChannel *io_stdin = g_io_channel_unix_new(fileno(stdin));
    app.keyboard_source_id = g_io_add_watch(io_stdin, G_IO_IN, (GIOFunc)_handle_keyboard, app.main_loop);
    g_io_channel_unref(io_stdin);

    g_print("Transcoding started. Press q<Enter> to stop.\n");
    g_main_loop_run(app.main_loop);

    /* Clean application. */
    _clean_app(&app);

    return 0;
}

You can also download it here.

Building

This source code along with the rest of tutorials can be compiled using the following commands.

On Linux:

mkdir fluendo-sdk-tutorials && cd fluendo-sdk-tutorials
meson /opt/fluendo-sdk/share/doc/fluendo-sdk/tutorials/src
ninja

On Windows:

mkdir fluendo-sdk-tutorials
cd fluendo-sdk-tutorials
meson C:\fluendo-sdk\<version>\<x86/x86_64>\share\doc\fluendo-sdk\tutorials\src
ninja

To generate a Visual Studio project, you can pass the --backend=vs option to meson.

Conclusions

To summarize, you have learned during this tutorial:

• How modifying the FluStreamVideoInfo and the FluStreamAudioInfo structs you can change the video and audio parameters, respectively