I want to transcode and down/re-sample the audio for output using ffmpeg's libav*/libswresample - I am using ffmpeg's (4.x) transcode_aac.c and resample_audio.c as reference - but the code produces audio with glitches that is clearly not what ffmpeg itself would produce (ie ffmpeg -i foo.wav -ar 22050 foo.m4a)
Based on the ffmpeg examples, to resample audio it appears that I need to set the output AVAudioContext and SwrContext sample_rate to what I desire and ensure the swr_convert() is provided with the correct number of output samples based av_rescale_rnd( swr_delay(), ...) once I have an decoded input audio. I've taken care to ensure all the relevant calculations of samples for output are taken into account in the merged code (below):
open_output_file() - AVCodecContext.sample_rate (avctx variable) set to our target (down sampled) sample_rate
read_decode_convert_and_store() is where the work happens: input audio is decoded to an AVFrame and this input frame is converted before being encoded.
init_converted_samples() and av_samples_alloc() uses the input frame's nb_samples
ADDED: calc the number of output samples via av_rescale_rnd() and swr_delay()
UPDATED: convert_samples() and swr_convert() uses the input frame's samples and our calculated output samples as parameters
However the resulting audio file is produced with audio glitches. Does the community know of any references for how transcode AND resample should be done or what is missing in this example?
/* compile and run:
gcc -I/usr/include/ffmpeg transcode-swr-aac.c -lavformat -lavutil -lavcodec -lswresample -lm
./a.out foo.wav foo.m4a
*/
/*
* Copyright (c) 2013-2018 Andreas Unterweger
*
* This file is part of FFmpeg.
... ...
*
* #example transcode_aac.c
* Convert an input audio file to AAC in an MP4 container using FFmpeg.
* Formats other than MP4 are supported based on the output file extension.
* #author Andreas Unterweger (xxxx#xxxxx.com)
*/
#include <stdio.h>
#include "libavformat/avformat.h"
#include "libavformat/avio.h"
#include "libavcodec/avcodec.h"
#include "libavutil/audio_fifo.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/channel_layout.h"
#include "libavutil/frame.h"
#include "libavutil/opt.h"
#include "libswresample/swresample.h"
#define OUTPUT_BIT_RATE 128000
#define OUTPUT_CHANNELS 2
static int open_input_file(const char *filename,
AVFormatContext **input_format_context,
AVCodecContext **input_codec_context)
{
AVCodecContext *avctx;
const AVCodec *input_codec;
const AVStream *stream;
int error;
if ((error = avformat_open_input(input_format_context, filename, NULL,
NULL)) < 0) {
fprintf(stderr, "Could not open input file '%s' (error '%s')\n",
filename, av_err2str(error));
*input_format_context = NULL;
return error;
}
if ((error = avformat_find_stream_info(*input_format_context, NULL)) < 0) {
fprintf(stderr, "Could not open find stream info (error '%s')\n",
av_err2str(error));
avformat_close_input(input_format_context);
return error;
}
if ((*input_format_context)->nb_streams != 1) {
fprintf(stderr, "Expected one audio input stream, but found %d\n",
(*input_format_context)->nb_streams);
avformat_close_input(input_format_context);
return AVERROR_EXIT;
}
stream = (*input_format_context)->streams[0];
if (!(input_codec = avcodec_find_decoder(stream->codecpar->codec_id))) {
fprintf(stderr, "Could not find input codec\n");
avformat_close_input(input_format_context);
return AVERROR_EXIT;
}
avctx = avcodec_alloc_context3(input_codec);
if (!avctx) {
fprintf(stderr, "Could not allocate a decoding context\n");
avformat_close_input(input_format_context);
return AVERROR(ENOMEM);
}
/* Initialize the stream parameters with demuxer information. */
error = avcodec_parameters_to_context(avctx, stream->codecpar);
if (error < 0) {
avformat_close_input(input_format_context);
avcodec_free_context(&avctx);
return error;
}
/* Open the decoder for the audio stream to use it later. */
if ((error = avcodec_open2(avctx, input_codec, NULL)) < 0) {
fprintf(stderr, "Could not open input codec (error '%s')\n",
av_err2str(error));
avcodec_free_context(&avctx);
avformat_close_input(input_format_context);
return error;
}
/* Set the packet timebase for the decoder. */
avctx->pkt_timebase = stream->time_base;
/* Save the decoder context for easier access later. */
*input_codec_context = avctx;
return 0;
}
static int open_output_file(const char *filename,
AVCodecContext *input_codec_context,
AVFormatContext **output_format_context,
AVCodecContext **output_codec_context)
{
AVCodecContext *avctx = NULL;
AVIOContext *output_io_context = NULL;
AVStream *stream = NULL;
const AVCodec *output_codec = NULL;
int error;
if ((error = avio_open(&output_io_context, filename,
AVIO_FLAG_WRITE)) < 0) {
fprintf(stderr, "Could not open output file '%s' (error '%s')\n",
filename, av_err2str(error));
return error;
}
if (!(*output_format_context = avformat_alloc_context())) {
fprintf(stderr, "Could not allocate output format context\n");
return AVERROR(ENOMEM);
}
(*output_format_context)->pb = output_io_context;
if (!((*output_format_context)->oformat = av_guess_format(NULL, filename,
NULL))) {
fprintf(stderr, "Could not find output file format\n");
goto cleanup;
}
if (!((*output_format_context)->url = av_strdup(filename))) {
fprintf(stderr, "Could not allocate url.\n");
error = AVERROR(ENOMEM);
goto cleanup;
}
if (!(output_codec = avcodec_find_encoder(AV_CODEC_ID_AAC))) {
fprintf(stderr, "Could not find an AAC encoder.\n");
goto cleanup;
}
/* Create a new audio stream in the output file container. */
if (!(stream = avformat_new_stream(*output_format_context, NULL))) {
fprintf(stderr, "Could not create new stream\n");
error = AVERROR(ENOMEM);
goto cleanup;
}
avctx = avcodec_alloc_context3(output_codec);
if (!avctx) {
fprintf(stderr, "Could not allocate an encoding context\n");
error = AVERROR(ENOMEM);
goto cleanup;
}
/* Set the basic encoder parameters.
* SET OUR DESIRED output sample_rate here
*/
avctx->channels = OUTPUT_CHANNELS;
avctx->channel_layout = av_get_default_channel_layout(OUTPUT_CHANNELS);
// avctx->sample_rate = input_codec_context->sample_rate;
avctx->sample_rate = 22050;
avctx->sample_fmt = output_codec->sample_fmts[0];
avctx->bit_rate = OUTPUT_BIT_RATE;
avctx->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
/* Set the sample rate for the container. */
stream->time_base.den = avctx->sample_rate;
stream->time_base.num = 1;
if ((*output_format_context)->oformat->flags & AVFMT_GLOBALHEADER)
avctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
if ((error = avcodec_open2(avctx, output_codec, NULL)) < 0) {
fprintf(stderr, "Could not open output codec (error '%s')\n",
av_err2str(error));
goto cleanup;
}
error = avcodec_parameters_from_context(stream->codecpar, avctx);
if (error < 0) {
fprintf(stderr, "Could not initialize stream parameters\n");
goto cleanup;
}
/* Save the encoder context for easier access later. */
*output_codec_context = avctx;
return 0;
cleanup:
avcodec_free_context(&avctx);
avio_closep(&(*output_format_context)->pb);
avformat_free_context(*output_format_context);
*output_format_context = NULL;
return error < 0 ? error : AVERROR_EXIT;
}
/**
* Initialize one data packet for reading or writing.
*/
static int init_packet(AVPacket **packet)
{
if (!(*packet = av_packet_alloc())) {
fprintf(stderr, "Could not allocate packet\n");
return AVERROR(ENOMEM);
}
return 0;
}
static int init_input_frame(AVFrame **frame)
{
if (!(*frame = av_frame_alloc())) {
fprintf(stderr, "Could not allocate input frame\n");
return AVERROR(ENOMEM);
}
return 0;
}
static int init_resampler(AVCodecContext *input_codec_context,
AVCodecContext *output_codec_context,
SwrContext **resample_context)
{
int error;
/**
* create the resample, including ref to the desired output sample rate
*/
*resample_context = swr_alloc_set_opts(NULL,
av_get_default_channel_layout(output_codec_context->channels),
output_codec_context->sample_fmt,
output_codec_context->sample_rate,
av_get_default_channel_layout(input_codec_context->channels),
input_codec_context->sample_fmt,
input_codec_context->sample_rate,
0, NULL);
if (!*resample_context < 0) {
fprintf(stderr, "Could not allocate resample context\n");
return AVERROR(ENOMEM);
}
if ((error = swr_init(*resample_context)) < 0) {
fprintf(stderr, "Could not open resample context\n");
swr_free(resample_context);
return error;
}
return 0;
}
static int init_fifo(AVAudioFifo **fifo, AVCodecContext *output_codec_context)
{
if (!(*fifo = av_audio_fifo_alloc(output_codec_context->sample_fmt,
output_codec_context->channels, 1))) {
fprintf(stderr, "Could not allocate FIFO\n");
return AVERROR(ENOMEM);
}
return 0;
}
static int write_output_file_header(AVFormatContext *output_format_context)
{
int error;
if ((error = avformat_write_header(output_format_context, NULL)) < 0) {
fprintf(stderr, "Could not write output file header (error '%s')\n",
av_err2str(error));
return error;
}
return 0;
}
static int decode_audio_frame(AVFrame *frame,
AVFormatContext *input_format_context,
AVCodecContext *input_codec_context,
int *data_present, int *finished)
{
AVPacket *input_packet;
int error;
error = init_packet(&input_packet);
if (error < 0)
return error;
*data_present = 0;
*finished = 0;
if ((error = av_read_frame(input_format_context, input_packet)) < 0) {
if (error == AVERROR_EOF)
*finished = 1;
else {
fprintf(stderr, "Could not read frame (error '%s')\n",
av_err2str(error));
goto cleanup;
}
}
if ((error = avcodec_send_packet(input_codec_context, input_packet)) < 0) {
fprintf(stderr, "Could not send packet for decoding (error '%s')\n",
av_err2str(error));
goto cleanup;
}
error = avcodec_receive_frame(input_codec_context, frame);
if (error == AVERROR(EAGAIN)) {
error = 0;
goto cleanup;
} else if (error == AVERROR_EOF) {
*finished = 1;
error = 0;
goto cleanup;
} else if (error < 0) {
fprintf(stderr, "Could not decode frame (error '%s')\n",
av_err2str(error));
goto cleanup;
} else {
*data_present = 1;
goto cleanup;
}
cleanup:
av_packet_free(&input_packet);
return error;
}
static int init_converted_samples(uint8_t ***converted_input_samples,
AVCodecContext *output_codec_context,
int frame_size)
{
int error;
if (!(*converted_input_samples = calloc(output_codec_context->channels,
sizeof(**converted_input_samples)))) {
fprintf(stderr, "Could not allocate converted input sample pointers\n");
return AVERROR(ENOMEM);
}
if ((error = av_samples_alloc(*converted_input_samples, NULL,
output_codec_context->channels,
frame_size,
output_codec_context->sample_fmt, 0)) < 0) {
fprintf(stderr,
"Could not allocate converted input samples (error '%s')\n",
av_err2str(error));
av_freep(&(*converted_input_samples)[0]);
free(*converted_input_samples);
return error;
}
return 0;
}
static int convert_samples(const uint8_t **input_data, const int input_nb_samples,
uint8_t **converted_data, const int output_nb_samples,
SwrContext *resample_context)
{
int error;
if ((error = swr_convert(resample_context,
converted_data, output_nb_samples,
input_data , input_nb_samples)) < 0) {
fprintf(stderr, "Could not convert input samples (error '%s')\n",
av_err2str(error));
return error;
}
return 0;
}
static int add_samples_to_fifo(AVAudioFifo *fifo,
uint8_t **converted_input_samples,
const int frame_size)
{
int error;
if ((error = av_audio_fifo_realloc(fifo, av_audio_fifo_size(fifo) + frame_size)) < 0) {
fprintf(stderr, "Could not reallocate FIFO\n");
return error;
}
if (av_audio_fifo_write(fifo, (void **)converted_input_samples,
frame_size) < frame_size) {
fprintf(stderr, "Could not write data to FIFO\n");
return AVERROR_EXIT;
}
return 0;
}
static int read_decode_convert_and_store(AVAudioFifo *fifo,
AVFormatContext *input_format_context,
AVCodecContext *input_codec_context,
AVCodecContext *output_codec_context,
SwrContext *resampler_context,
int *finished)
{
AVFrame *input_frame = NULL;
uint8_t **converted_input_samples = NULL;
int data_present;
int ret = AVERROR_EXIT;
if (init_input_frame(&input_frame))
goto cleanup;
if (decode_audio_frame(input_frame, input_format_context,
input_codec_context, &data_present, finished))
goto cleanup;
if (*finished) {
ret = 0;
goto cleanup;
}
if (data_present) {
/* Initialize the temporary storage for the converted input samples. */
if (init_converted_samples(&converted_input_samples, output_codec_context,
input_frame->nb_samples))
goto cleanup;
/* figure out how many samples are required for target sample_rate incl
* any items left in the swr buffer
*/
int output_nb_samples = av_rescale_rnd(
swr_get_delay(resampler_context, input_codec_context->sample_rate) + input_frame->nb_samples,
output_codec_context->sample_rate,
input_codec_context->sample_rate,
AV_ROUND_UP);
/* ignore, just to ensure we've got enough buffer alloc'd for conversion buffer */
av_assert1(input_frame->nb_samples > output_nb_samples);
/* Convert the input samples to the desired output sample format, via swr_convert().
*/
if (convert_samples((const uint8_t**)input_frame->extended_data, input_frame->nb_samples,
converted_input_samples, output_nb_samples,
resampler_context))
goto cleanup;
/* Add the converted input samples to the FIFO buffer for later processing. */
if (add_samples_to_fifo(fifo, converted_input_samples,
output_nb_samples))
goto cleanup;
ret = 0;
}
ret = 0;
cleanup:
if (converted_input_samples) {
av_freep(&converted_input_samples[0]);
free(converted_input_samples);
}
av_frame_free(&input_frame);
return ret;
}
static int init_output_frame(AVFrame **frame,
AVCodecContext *output_codec_context,
int frame_size)
{
int error;
if (!(*frame = av_frame_alloc())) {
fprintf(stderr, "Could not allocate output frame\n");
return AVERROR_EXIT;
}
/* Set the frame's parameters, especially its size and format.
* av_frame_get_buffer needs this to allocate memory for the
* audio samples of the frame.
* Default channel layouts based on the number of channels
* are assumed for simplicity. */
(*frame)->nb_samples = frame_size;
(*frame)->channel_layout = output_codec_context->channel_layout;
(*frame)->format = output_codec_context->sample_fmt;
(*frame)->sample_rate = output_codec_context->sample_rate;
/* Allocate the samples of the created frame. This call will make
* sure that the audio frame can hold as many samples as specified. */
if ((error = av_frame_get_buffer(*frame, 0)) < 0) {
fprintf(stderr, "Could not allocate output frame samples (error '%s')\n",
av_err2str(error));
av_frame_free(frame);
return error;
}
return 0;
}
/* Global timestamp for the audio frames. */
static int64_t pts = 0;
/**
* Encode one frame worth of audio to the output file.
*/
static int encode_audio_frame(AVFrame *frame,
AVFormatContext *output_format_context,
AVCodecContext *output_codec_context,
int *data_present)
{
AVPacket *output_packet;
int error;
error = init_packet(&output_packet);
if (error < 0)
return error;
/* Set a timestamp based on the sample rate for the container. */
if (frame) {
frame->pts = pts;
pts += frame->nb_samples;
}
*data_present = 0;
error = avcodec_send_frame(output_codec_context, frame);
if (error < 0 && error != AVERROR_EOF) {
fprintf(stderr, "Could not send packet for encoding (error '%s')\n",
av_err2str(error));
goto cleanup;
}
error = avcodec_receive_packet(output_codec_context, output_packet);
if (error == AVERROR(EAGAIN)) {
error = 0;
goto cleanup;
} else if (error == AVERROR_EOF) {
error = 0;
goto cleanup;
} else if (error < 0) {
fprintf(stderr, "Could not encode frame (error '%s')\n",
av_err2str(error));
goto cleanup;
} else {
*data_present = 1;
}
/* Write one audio frame from the temporary packet to the output file. */
if (*data_present &&
(error = av_write_frame(output_format_context, output_packet)) < 0) {
fprintf(stderr, "Could not write frame (error '%s')\n",
av_err2str(error));
goto cleanup;
}
cleanup:
av_packet_free(&output_packet);
return error;
}
/**
* Load one audio frame from the FIFO buffer, encode and write it to the
* output file.
*/
static int load_encode_and_write(AVAudioFifo *fifo,
AVFormatContext *output_format_context,
AVCodecContext *output_codec_context)
{
AVFrame *output_frame;
/* Use the maximum number of possible samples per frame.
* If there is less than the maximum possible frame size in the FIFO
* buffer use this number. Otherwise, use the maximum possible frame size. */
const int frame_size = FFMIN(av_audio_fifo_size(fifo),
output_codec_context->frame_size);
int data_written;
if (init_output_frame(&output_frame, output_codec_context, frame_size))
return AVERROR_EXIT;
/* Read as many samples from the FIFO buffer as required to fill the frame.
* The samples are stored in the frame temporarily. */
if (av_audio_fifo_read(fifo, (void **)output_frame->data, frame_size) < frame_size) {
fprintf(stderr, "Could not read data from FIFO\n");
av_frame_free(&output_frame);
return AVERROR_EXIT;
}
/* Encode one frame worth of audio samples. */
if (encode_audio_frame(output_frame, output_format_context,
output_codec_context, &data_written)) {
av_frame_free(&output_frame);
return AVERROR_EXIT;
}
av_frame_free(&output_frame);
return 0;
}
/**
* Write the trailer of the output file container.
*/
static int write_output_file_trailer(AVFormatContext *output_format_context)
{
int error;
if ((error = av_write_trailer(output_format_context)) < 0) {
fprintf(stderr, "Could not write output file trailer (error '%s')\n",
av_err2str(error));
return error;
}
return 0;
}
int main(int argc, char **argv)
{
AVFormatContext *input_format_context = NULL, *output_format_context = NULL;
AVCodecContext *input_codec_context = NULL, *output_codec_context = NULL;
SwrContext *resample_context = NULL;
AVAudioFifo *fifo = NULL;
int ret = AVERROR_EXIT;
if (argc != 3) {
fprintf(stderr, "Usage: %s <input file> <output file>\n", argv[0]);
exit(1);
}
if (open_input_file(argv[1], &input_format_context,
&input_codec_context))
goto cleanup;
if (open_output_file(argv[2], input_codec_context,
&output_format_context, &output_codec_context))
goto cleanup;
if (init_resampler(input_codec_context, output_codec_context,
&resample_context))
goto cleanup;
if (init_fifo(&fifo, output_codec_context))
goto cleanup;
if (write_output_file_header(output_format_context))
goto cleanup;
while (1) {
/* Use the encoder's desired frame size for processing. */
const int output_frame_size = output_codec_context->frame_size;
int finished = 0;
while (av_audio_fifo_size(fifo) < output_frame_size) {
/* Decode one frame worth of audio samples, convert it to the
* output sample format and put it into the FIFO buffer. */
if (read_decode_convert_and_store(fifo, input_format_context,
input_codec_context,
output_codec_context,
resample_context, &finished))
goto cleanup;
if (finished)
break;
}
while (av_audio_fifo_size(fifo) >= output_frame_size ||
(finished && av_audio_fifo_size(fifo) > 0))
if (load_encode_and_write(fifo, output_format_context,
output_codec_context))
goto cleanup;
if (finished) {
int data_written;
do {
if (encode_audio_frame(NULL, output_format_context,
output_codec_context, &data_written))
goto cleanup;
} while (data_written);
break;
}
}
if (write_output_file_trailer(output_format_context))
goto cleanup;
ret = 0;
cleanup:
if (fifo)
av_audio_fifo_free(fifo);
swr_free(&resample_context);
if (output_codec_context)
avcodec_free_context(&output_codec_context);
if (output_format_context) {
avio_closep(&output_format_context->pb);
avformat_free_context(output_format_context);
}
if (input_codec_context)
avcodec_free_context(&input_codec_context);
if (input_format_context)
avformat_close_input(&input_format_context);
return ret;
}
After going through the ffmpeg/libav mailing list, particularly https://ffmpeg.org/pipermail/libav-user/2017-July/010496.html, I was able to modify the ffmpeg transcode_aac.c example to perform the sample rate conversion.
In the original code, the main loop reads/decode/covert/store in one function before passing the samples to a AVAudioFifo which is used by the encoder.
Some encoders expects a specific number of samples - if you provide less, it appears the encoder pads up to expected and this results in the glitches mentioned in my first attempt.
The key, as per the ffmpeg mailing list, is to buffer / concat the decoded input samples until we have enough samples for at least one frame for the encoder. To do this we split the read/decode from the convert/store with the read/decode data being stored in a new intermediary AVAudioFifo. Once the intermediary fifo has enough samples, they get converted and the output is added to the original fifo.
static int read_decode_and_store(AVAudioFifo *fifo,
AVFormatContext *input_format_context,
AVCodecContext *input_codec_context,
const int audio_stream_idx,
int *finished)
{
AVFrame *input_frame = NULL;
int data_present = 0;
int ret = AVERROR_EXIT;
if (init_input_frame(&input_frame))
goto cleanup;
if (decode_audio_frame(input_frame, input_format_context,
input_codec_context, audio_stream_idx, &data_present, finished))
goto cleanup;
if (*finished) {
ret = 0;
goto cleanup;
}
if (data_present) {
/* Add the converted input samples to the FIFO buffer for later processing. */
if (add_samples_to_fifo(fifo, (uint8_t**)input_frame->extended_data, input_frame->nb_samples))
goto cleanup;
}
ret = 0;
cleanup:
av_frame_free(&input_frame);
return ret;
}
static int load_convert_and_store(AVAudioFifo* output_samples_fifo, const AVFormatContext* output_context, AVCodecContext* output_codec_context, int output_frame_size,
AVAudioFifo* input_samples_fifo, const AVFormatContext* input_context, AVCodecContext* input_codec_context,
SwrContext* resample_context)
{
uint8_t **converted_input_samples = NULL;
int ret = AVERROR_EXIT;
AVFrame *input_frame;
const int frame_size = FFMIN(av_audio_fifo_size(input_samples_fifo),
output_frame_size);
// yes this is init_output_frame
if (init_output_frame(&input_frame, input_codec_context, frame_size))
return AVERROR_EXIT;
if (av_audio_fifo_read(input_samples_fifo, (void **)input_frame->data, frame_size) < frame_size) {
fprintf(stderr, "Could not read data from input samples FIFO");
av_frame_free(&input_frame);
return AVERROR_EXIT;
}
int nb_samples = (output_codec_context->sample_rate == input_codec_context->sample_rate) ?
input_frame->nb_samples :
av_rescale_rnd(swr_get_delay(resample_context, input_codec_context->sample_rate) + input_frame->nb_samples, output_codec_context->sample_rate, input_codec_context->sample_rate, AV_ROUND_UP);
if (init_converted_samples(&converted_input_samples, output_codec_context,
nb_samples))
goto cleanup;
/* **** Modify convert_samples() to return the value from swr_convert() **** */
if ( (nb_samples = convert_samples((const uint8_t**)input_frame->extended_data, input_frame->nb_samples,
converted_input_samples, output_codec_context->frame_size,
resample_context)) < 0)
goto cleanup;
if (add_samples_to_fifo(output_samples_fifo, converted_input_samples, nb_samples))
goto cleanup;
ret = 0;
cleanup:
if (converted_input_samples) {
av_freep(&converted_input_samples[0]);
free(converted_input_samples);
}
av_frame_free(&input_frame);
return ret;
}
int main()
{
...
while (1)
{
const int output_frame_size = output_codec_context->frame_size;
int finished = 0;
/* Re: Resample frame to specified number of samples
* https://ffmpeg.org/pipermail/libav-user/2017-July/010496.html
* Yes, you need to buffer sufficient audio frames to feed to the encoder.
*
* Calculate the number of in samples:
in_nb_samples = av_rescale_rnd(swr_get_delay(swr_ctx, c->sample_rate) +
out_nb_samples,
in_sample_rate, c->sample_rate, AV_ROUND_DOWN);
then allocate buffers to concatenate the in samples until you have enough
to pass to swr_ctx.
*/
while (av_audio_fifo_size(input_samples_fifo) < output_frame_size) {
if (read_decode_and_store(input_samples_fifo,
input_format_context, input_codec_context,
audio_stream_idx,
&finished))
goto cleanup;
if (finished)
break;
}
while (av_audio_fifo_size(input_samples_fifo) >= output_frame_size ||
(finished && av_audio_fifo_size(input_samples_fifo) > 0)) {
/* take all input samples and convert them before handing off to encoder
*/
if (load_convert_and_store(fifo,
output_format_context, output_codec_context, output_frame_size,
input_samples_fifo, input_format_context, input_codec_context,
resample_context))
goto cleanup;
}
}
/* If we have enough samples for the encoder, we encode them.
* At the end of the file, we pass the remaining samples to
* the encoder. */
.... // existing code
}
I'm trying to develop a simple audio program that runs on a ADSL router. I found two GPL sources for other routers that have the same chips and I'm able to build kernel and rootfs. I already add ALSA support to the kernel and successfully deployed to the router.
Now I'm trying to get my program to work. Using a usb sound card, the first thing that I'm doing is to passthrough capture signal to the playback and measure the latency.
I already remove all jitter that was present but I can't get a better latency that 1 second (it's a lot of latency). I'm initializing the device with 128 frames of buffer and 1 frame for period size, and latency is always 1 second. I'm using only 1 channel and a rate of 44100, but also tested with a rate of 8000hz and the latency is the same
I already tried setting another priority to the running thread and tested a lot of combinations of hw params, but never could get minor latency than that.
I checked the cpu and memory while my program is running and there is a lot of free resources. I removed many services of the system like telnet, samba and other programs that could interfere with my program.
An interesting point is that when my program only set the playback device and play some pcm file, there is no latency (right after I start the program, the sound is played correctly).
Linux kernel version is v2.6.22.15. Alsa driver is 1.0.14. I tried a cheap usb sound card, a Zoom h1 recorder (that works as usb card) and a Presonus 22vsl audio interface, always the same latency.
The CPU is a Ralink rt63365, it has 4 cores and it seems to run at 400hz aprox.
I don't know what else to try. What kind of test could I do to detect where the latency problem is generated?
Edit
I forget to mention that I run the same program with all the cards mentioned but in my PC (kernel version > 5, with the respective alsa version) and the program works perfectly, there is no notable latency.
Main functions of my program:
void setscheduler(void)
{
struct sched_param sched_param;
if (sched_getparam(0, &sched_param) < 0) {
printf("Scheduler getparam failed...\n");
return;
}
sched_param.sched_priority = sched_get_priority_max(SCHED_RR);
if (!sched_setscheduler(0, SCHED_RR, &sched_param)) {
printf("Scheduler set to Round Robin with priority %i...\n", sched_param.sched_priority);
fflush(stdout);
return;
}
printf("!!!Scheduler set to Round Robin with priority %i FAILED!!!\n", sched_param.sched_priority);
}
int setup_alsa_handle(snd_pcm_t *handle, snd_pcm_stream_t stream)
{
int err = 0;
snd_pcm_hw_params_t *hw_params;
snd_pcm_sw_params_t *sw_params;
if ((err = snd_pcm_hw_params_malloc(&hw_params)) < 0)
{
fprintf(stderr, "cannot allocate hardware parameter structure (%s)\n",
snd_strerror(err));
exit(1);
}
if ((err = snd_pcm_hw_params_any(handle, hw_params)) < 0)
{
fprintf(stderr, "cannot initialize hardware parameter structure (%s)\n",
snd_strerror(err));
exit(1);
}
if ((err = snd_pcm_hw_params_set_access(handle, hw_params,
SND_PCM_ACCESS_RW_INTERLEAVED)) < 0)
{
fprintf(stderr, "cannot set access type (%s)\n", snd_strerror(err));
exit(1);
}
if ((err = snd_pcm_hw_params_set_format(handle, hw_params, AUDIO_FORMAT)) < 0)
{
fprintf(stderr, "cannot set sample format (%s)\n", snd_strerror(err));
exit(1);
}
unsigned int rate = SAMPLE_RATE;
if ((err = snd_pcm_hw_params_set_rate_near(handle, hw_params, &rate, 0)) <
0)
{
fprintf(stderr, "cannot set sample rate (%s)\n", snd_strerror(err));
exit(1);
}
int dir;
snd_pcm_uframes_t period_size = PERIOD_SIZE_IN_FRAMES;
if ((err = snd_pcm_hw_params_set_period_size_near(handle, hw_params, &period_size, &dir)) <
0)
{
fprintf(stderr, "cannot set period size (%s)\n", snd_strerror(err));
exit(1);
}
int channels;
if (stream == SND_PCM_STREAM_CAPTURE)
{
channels = CAPTURE_CHANNELS;
}
else
{
channels = PLAYBACK_CHANNELS;
}
if ((err = snd_pcm_hw_params_set_channels(handle, hw_params, channels)) < 0)
{
fprintf(stderr, "cannot set channel count (%s)\n", snd_strerror(err));
exit(1);
}
if ((err = snd_pcm_hw_params(handle, hw_params)) < 0)
{
fprintf(stderr, "cannot set parameters (%s)\n", snd_strerror(err));
exit(1);
}
snd_pcm_hw_params_free(hw_params);
if ((err = snd_pcm_sw_params_malloc(&sw_params)) < 0) {
fprintf(stderr, "cannot allocate software parameters structure(%s)\n",
snd_strerror(err));
return err;
}
if ((err = snd_pcm_sw_params_current(handle, sw_params)) < 0) {
fprintf(stderr, "cannot initialize software parameters structure(%s)\n",
snd_strerror(err));
return err;
}
if ((err = snd_pcm_sw_params_set_avail_min(handle, sw_params, FRAMES_PER_BUFFER)) < 0) {
fprintf(stderr, "cannot set minimum available count(%s)\n",
snd_strerror(err));
return err;
}
if ((err = snd_pcm_sw_params_set_start_threshold(handle, sw_params, 0U)) < 0) {
fprintf(stderr, "cannot set start mode(%s)\n",
snd_strerror(err));
return err;
}
if ((err = snd_pcm_sw_params(handle, sw_params)) < 0) {
fprintf(stderr, "cannot set software parameters(%s)\n",
snd_strerror(err));
return err;
}
return 0;
}
void start_stream(snd_pcm_t *_playback_handle, snd_pcm_t *_capture_handle,
void (*stream_callback)(const void *, void *, unsigned long,
void *),
void (*controls_callback)(void *), void *data)
{
audio_sample *in_buffer;
audio_sample *out_buffer;
FILE *fout = NULL;
int in_result = 0;
int out_result = 0;
int temp_n = 0;
int buffer_frames = FRAMES_PER_BUFFER;
int frame_size_in_bytes = snd_pcm_format_width(AUDIO_FORMAT) / 8;
int in_buffer_frames = buffer_frames * CAPTURE_CHANNELS;
in_buffer = (audio_sample *)malloc(in_buffer_frames * frame_size_in_bytes);
int out_buffer_frames = buffer_frames * PLAYBACK_CHANNELS;
out_buffer = (audio_sample *)malloc(out_buffer_frames * frame_size_in_bytes);
memset(in_buffer, SAMPLE_SILENCE, in_buffer_frames * frame_size_in_bytes);
memset(out_buffer, SAMPLE_SILENCE, out_buffer_frames * frame_size_in_bytes);
int err;
while (1)
{
int avail;
if ((err = snd_pcm_wait(_playback_handle, 1000)) < 0) {
fprintf(stderr, "poll failed(%s)\n", strerror(errno));
break;
}
avail = snd_pcm_avail_update(_capture_handle);
fprintf(stderr, "1 avail (%d)\n", avail);
if (avail > 0) {
if (avail > FRAMES_PER_BUFFER)
avail = FRAMES_PER_BUFFER;
snd_pcm_readi(_capture_handle, in_buffer, avail);
}
avail = snd_pcm_avail_update(_playback_handle);
fprintf(stderr, "2 avail (%d)\n", avail);
if (avail > 0) {
if (avail > FRAMES_PER_BUFFER)
avail = FRAMES_PER_BUFFER;
snd_pcm_writei(_playback_handle, in_buffer, avail);
}
}
}
/**
* Main
*/
int main(int argc, char *argv[])
{
setscheduler();
if ((err = snd_pcm_open(&capture_handle, CAPTURE_AUDIO_DEVICE, SND_PCM_STREAM_CAPTURE, 0)) < 0)
{
fprintf(stderr, "cannot open audio device '%s'. Error: %s\n", CAPTURE_AUDIO_DEVICE, snd_strerror(err));
exit(1);
}
setup_alsa_handle(capture_handle, SND_PCM_STREAM_CAPTURE);
// Init playback device
if ((err = snd_pcm_open(&playback_handle, PLAYBACK_AUDIO_DEVICE, SND_PCM_STREAM_PLAYBACK, 0)) < 0)
{
fprintf(stderr, "cannot open audio device '%s'. Error: %s\n", PLAYBACK_AUDIO_DEVICE, snd_strerror(err));
exit(1);
}
setup_alsa_handle(playback_handle, SND_PCM_STREAM_PLAYBACK);
if ((err = snd_pcm_start(capture_handle)) < 0) {
fprintf(stderr, "cannot prepare audio interface for use(%s)\n",
snd_strerror(err));
return err;
}
if ((err = snd_pcm_prepare(playback_handle)) < 0) {
fprintf(stderr, "cannot prepare audio interface for use(%s)\n",
snd_strerror(err));
return err;
}
// Start stream
start_stream(playback_handle, capture_handle, audio_processing_callback, controls_callback, &data, write_to_file, read_from_file);
return 0;
}
I am developing this module for custom device that, in fact, a 4*8-bit i-o ports attached to ISA bus with addresses 0x0120 - 0x0123. This driver is based on "scull" by Alessandro Rubini and Jonathan Corbet. My OS is Ubuntu 10.04, kernel is 2.6.32-74 generic, I use built-in console-oriented compiler gcc.
While inserting compiled module using "insmod" I get an error "bad address" and module was not loaded. I've tried to debug it using "printk" and found out that my module successfully gets a range of i-o ports, major and minor numbers and then, when trying to do "Reset_Port" function it generates an error "bad address" and exits.
Can anybody tell me, what am I doing wrong?
Here are __exit and __init functions of my module
void __exit ET3201_exit(void)
{
int i;
dev_t devno = MKDEV(ET3201_major, ET3201_minor);
/* Get rid of our char dev entries */
if (ET3201_devices) {
for (i = 0; i < ET3201_nr_devs; i++) {
ET3201_trim(ET3201_devices + i);
cdev_del(&ET3201_devices[i].cdev);
}
kfree(ET3201_devices);
}
#ifdef ET3201_DEBUG /* use proc only if debugging */
ET3201_remove_proc();
#endif
/* cleanup_module is never called if registering failed */
unregister_chrdev_region(devno, ET3201_nr_devs);
if ( ! port ) release_region(BaseIO, 8);
printk(KERN_INFO "Goodbye, cruel world - ET3201 is unloaded\n");
/* and call the cleanup functions for friend devices */
/*ET3201_access_cleanup();*/
}
/*----------------------------------------------------------------------------*/
/* Set up the char_dev structure for this device. */
static void ET3201_setup_cdev(struct ET3201_dev *dev, int index)
{
int err, devno = MKDEV(ET3201_major, ET3201_minor + index);
cdev_init(&dev->cdev, &ET3201_fops);
dev->cdev.owner = THIS_MODULE;
dev->cdev.ops = &ET3201_fops;
dev->CAMAC_Module_Number = CAMAC_Nmod;
dev->CAMAC_Command_Adress = CAMAC_Adcom;
dev->Driver_Number = ET3201_minor + index;
err = cdev_add (&dev->cdev, devno, 1);
/* Fail gracefully if need be */
if (err)
printk(KERN_NOTICE "Error %d adding ET3201%d", err, index);
}
/*----------------------------------------------------------------------------*/
int __init ET3201_init(void)
{
int result = 0;
int i;
dev_t dev = 0;
BaseIO = Base;
/* Get a range of minor numbers to work with, asking for a dynamic
major unless directed otherwise at load time. */
if (ET3201_major) {
dev = MKDEV(ET3201_major, ET3201_minor);
result = register_chrdev_region(dev, ET3201_nr_devs, "ET3201");
} else {
result = alloc_chrdev_region(&dev, ET3201_minor, ET3201_nr_devs, "ET3201");
ET3201_major = MAJOR(dev);
}
if (result < 0) {
printk(KERN_WARNING "ET3201: can't get major %d\n", ET3201_major);
return result;
}
port = request_region(BaseIO, 8, "ET3201");
if ( port == NULL ) {
printk(KERN_WARNING "ET3201 cannot reserve i-o ports %lu \n", BaseIO);
return -ENODEV;
goto fail;
}
/*
* allocate the devices -- we can't have them static, as the number
* can be specified at load time
*/
ET3201_devices = kmalloc(ET3201_nr_devs * sizeof(struct ET3201_dev), GFP_KERNEL);
if (! ET3201_devices) {
result = -ENOMEM;
printk(KERN_ALERT "ET3201: can't get memory \n");
goto fail; /* Fail gracefully if need be */
}
memset(ET3201_devices, 0, ET3201_nr_devs * sizeof(struct ET3201_dev));
/* Initialize each device. */
for (i = 0; i < ET3201_nr_devs; i++) {
ET3201_devices[i].quantum = ET3201_quantum;
ET3201_devices[i].qset = ET3201_qset;
init_MUTEX(&ET3201_devices[i].sem);
ET3201_setup_cdev(&ET3201_devices[i], i);
}
/* At this point call the init function for any friend device */
dev = MKDEV(ET3201_major, ET3201_minor + ET3201_nr_devs);
/*dev += ET3201_access_init(dev);*/
printk(KERN_INFO "ET3201 is initialized with major %d\n", ET3201_major);
if ( port != NULL ){
printk(KERN_INFO "ET3201 is trying to reset %d devices\n", ET3201_nr_devs);
result = Reset_Port();
}
if ( result != 0 ) {
printk(KERN_ALERT "ET3201: device cannot reset with result %d\n", result);
result = -EFAULT;
goto fail;
}
#ifdef ET3201_DEBUG /* only when debugging */
ET3201_create_proc();
#endif
return 0; /* succeed */
fail:
ET3201_exit();
return result;
}
/*----------------------------------------------------------------------------*/
module_init(ET3201_init);
module_exit(ET3201_exit);
MODULE_LICENSE("GPL");
MODULE_ALIAS_MISCDEV(ET3201_minor);
and next will be Reset_Port()
static int Reset_Port(void)
{
int result = -EIO;
int count;
if (port == NULL) goto fail;
for ( count = 0; count < ET3201_nr_devs; count++ )
{
outb(0x00, ports[count]);
}
wmb(); /*write memory barrier*/
LastOp = E_Reset;
result = 0; /* success */
fail:
return result;
}
EXPORT_SYMBOL(Reset_Port);
Now, after fixing 'int Reset_Port(void)' I've got another problem -
'WARNING: modpost: Found 1 section mismatch(es).'
After debugging I see that this is a result of calling 'ET3201_exit()'
from 'module_init()' - when I remarked this call, warning disappeared.
Surprising that exactly the same call was made in "scull" driver of respected authors - and it works.
Question: What can lead to kernel mismatch in this code?
Yes! The bug is fixed - after declaring ' int Reset_Port(void) ' the module was inserted and removed successfully. I thought,(but it was wrong) that all functions that can be called from within ' module_init() ' must not be declared as static.
I create a file:
m_fileHandle = CreateFileA(
m_pszFilename,
GENERIC_WRITE,
0,
NULL,
CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH,
NULL);
Then write to it:
const BOOL bSuccess = WriteFile(
m_fileHandle,
buffer,
dataSize,
&tempBytesWritten,
NULL );
When I start the program, WriteFile fails and GetLastError() returns error 87.
I read that WriteFile on a file created with flag FILE_FLAG_NO_BUFFERING fails when dataSize is not a multiple of hard disk sector size.
If that is the reason for the error, then why does the code work fine when I debug in Visual Studio Express 2012?
Solution was here: File Buffering https://msdn.microsoft.com/en-us/library/windows/desktop/cc644950%28v=vs.85%29.aspx
Working code:
#include "stdafx.h"
#include "assert.h"
#include <iostream>
#include <Windows.h>
#include <comutil.h>
using namespace std;
namespace{
unsigned long tempBytesWritten = 0;
HANDLE m_fileHandle;
char m_pszFilename[_MAX_PATH] = "";
// Create a temporary file for benchmark
int createFile()
{
WCHAR tempPath[MAX_PATH];
GetTempPath(_countof(tempPath), tempPath);
_bstr_t p(tempPath);
const char* c = p;
strcpy(m_pszFilename, c);
strcat(m_pszFilename, "testRawFile.raw");
cout << "Writing to " << m_pszFilename << endl;
m_fileHandle = CreateFileA(
m_pszFilename,
GENERIC_WRITE,
0,
NULL,
CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH,
NULL);
if (m_fileHandle == INVALID_HANDLE_VALUE)
{
assert( false );
}
return 0;
}
}
DWORD DetectSectorSize( WCHAR * devName, PSTORAGE_ACCESS_ALIGNMENT_DESCRIPTOR pAlignmentDescriptor)
{
DWORD Bytes = 0;
BOOL bReturn = FALSE;
DWORD Error = NO_ERROR;
STORAGE_PROPERTY_QUERY Query;
ZeroMemory(&Query, sizeof(Query));
HANDLE hFile = CreateFileW( devName,
STANDARD_RIGHTS_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
NULL);
if (hFile==INVALID_HANDLE_VALUE) {
wprintf(L" hFile==INVALID_HANDLE_VALUE. GetLastError() returns %lu.\n", Error=GetLastError());
return Error;
}
Query.QueryType = PropertyStandardQuery;
Query.PropertyId = StorageAccessAlignmentProperty;
bReturn = DeviceIoControl( hFile,
IOCTL_STORAGE_QUERY_PROPERTY,
&Query,
sizeof(STORAGE_PROPERTY_QUERY),
pAlignmentDescriptor,
sizeof(STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR),
&Bytes,
NULL);
if (bReturn == FALSE) {
wprintf(L" bReturn==FALSE. GetLastError() returns %lu.\n", Error=GetLastError());
}
CloseHandle(hFile);
return Error;
}
int main()
{
unsigned int dataSize = 2000;
DWORD Error = NO_ERROR;
STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR Alignment = {0};
// WCHAR szDisk[] = L"\\\\.\\PhysicalDrive0";
WCHAR szDisk[] = L"\\\\.\\C:";
Error = DetectSectorSize(szDisk, &Alignment);
if (Error) {
wprintf(L"Error %lu encountered while querying alignment.\n", Error);
return Error;
}
wprintf(L"Disk %s Properties\n", (WCHAR*) szDisk);
if (Alignment.BytesPerLogicalSector < Alignment.BytesPerPhysicalSector) {
wprintf(L" Emulated sector size is %lu bytes.\n", Alignment.BytesPerLogicalSector);
}
wprintf(L" Physical sector size is %lu bytes.\n", Alignment.BytesPerPhysicalSector);
dataSize = ((unsigned int)(dataSize + Alignment.BytesPerPhysicalSector - 1)/Alignment.BytesPerPhysicalSector) * Alignment.BytesPerPhysicalSector;
// Allocate buffer for file
unsigned char *buffer = new unsigned char[dataSize];
// Create file to write to
if ( createFile() != 0 )
{
printf("There was error creating the files... press Enter to exit.");
getchar();
return -1;
}
const BOOL bSuccess = WriteFile(m_fileHandle, buffer, dataSize, &tempBytesWritten, NULL );
if (!bSuccess)
{
cout << "Write failed with error " << GetLastError() << endl;
}
// clean up and remove file
CloseHandle(m_fileHandle);
wchar_t wtext[_MAX_PATH];
mbstowcs(wtext, m_pszFilename, strlen(m_pszFilename)+1);
DeleteFile(wtext);
return 0;
}