Convert .m4a to PCM using libavcodec - audio

I'm trying to convert a .m4a file to raw PCM file so that I can play it back in Audacity.
According to the AVCodecContext it is a 44100 Hz track using the sample format AV_SAMPLE_FMT_FLTP which, to my understanding, when decodeded using avcodec_decode_audio4, I should get two arrays of floating point values (one for each channel).
I'm unsure of the significance of the AVCodecContext's bits_per_coded_sample = 16
Unfortunately Audacity plays the result back as if I have the original track is mixed in with some white noise.
Here is some sample code of what I've been done. Note that I've also added a case for a track that uses signed 16bit non-interleaved data (sample_format = AC_SAMPLE_FMT_S16P), which Audacity plays back fine.
int AudioDecoder::decode(std::string path)
{
const char* input_filename=path.c_str();
av_register_all();
AVFormatContext* container=avformat_alloc_context();
if(avformat_open_input(&container,input_filename,NULL,NULL)<0){
printf("Could not open file");
}
if(avformat_find_stream_info(container, NULL)<0){
printf("Could not find file info");
}
av_dump_format(container,0,input_filename,false);
int stream_id=-1;
int i;
for(i=0;i<container->nb_streams;i++){
if(container->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO){
stream_id=i;
break;
}
}
if(stream_id==-1){
printf("Could not find Audio Stream");
}
AVDictionary *metadata=container->metadata;
AVCodecContext *ctx=container->streams[stream_id]->codec;
AVCodec *codec=avcodec_find_decoder(ctx->codec_id);
if(codec==NULL){
printf("cannot find codec!");
}
if(avcodec_open2(ctx,codec,NULL)<0){
printf("Codec cannot be found");
}
AVSampleFormat sfmt = ctx->sample_fmt;
AVPacket packet;
av_init_packet(&packet);
AVFrame *frame = avcodec_alloc_frame();
int buffer_size = AVCODEC_MAX_AUDIO_FRAME_SIZE+ FF_INPUT_BUFFER_PADDING_SIZE;;
uint8_t buffer[buffer_size];
packet.data=buffer;
packet.size =buffer_size;
FILE *outfile = fopen("test.raw", "wb");
int len;
int frameFinished=0;
while(av_read_frame(container,&packet) >= 0)
{
if(packet.stream_index==stream_id)
{
//printf("Audio Frame read \n");
int len=avcodec_decode_audio4(ctx, frame, &frameFinished, &packet);
if(frameFinished)
{
if (sfmt==AV_SAMPLE_FMT_S16P)
{ // Audacity: 16bit PCM little endian stereo
int16_t* ptr_l = (int16_t*)frame->extended_data[0];
int16_t* ptr_r = (int16_t*)frame->extended_data[1];
for (int i=0; i<frame->nb_samples; i++)
{
fwrite(ptr_l++, sizeof(int16_t), 1, outfile);
fwrite(ptr_r++, sizeof(int16_t), 1, outfile);
}
}
else if (sfmt==AV_SAMPLE_FMT_FLTP)
{ //Audacity: big endian 32bit stereo start offset 7 (but has noise)
float* ptr_l = (float*)frame->extended_data[0];
float* ptr_r = (float*)frame->extended_data[1];
for (int i=0; i<frame->nb_samples; i++)
{
fwrite(ptr_l++, sizeof(float), 1, outfile);
fwrite(ptr_r++, sizeof(float), 1, outfile);
}
}
}
}
}
fclose(outfile);
av_close_input_file(container);
return 0;
}
I'm hoping I've just done a naive conversion (most/less significant bit issues), but at present I've been unable to figure it out. Note that Audacity can only import RAW float data if its 32bit or 64 bit float (big or little endian).
Thanks for any insight.

I think problem is in "nb_samples". It's not exactly you need. It's better to try with "linesize[0]".
Example:
char* ptr_l = (char*)frame->extended_data[0];
char* ptr_r = (char*)frame->extended_data[1];
size_t size = sizeof(float);
for (int i=0; i<frame->linesize[0]; i+=size)
{
fwrite(ptr_l, size, 1, outfile);
fwrite(ptr_r, size, 1, outfile);
ptr_l += size;
ptr_r += size;
}
It's for "float", and repeat the same for "int16_t". But "size" will be "sizeof(int16_t)"

You must use a converter of AV_SAMPLE_FMT_FLTP in AC_SAMPLE_FMT_S16P
How to convert sample rate from AV_SAMPLE_FMT_FLTP to AV_SAMPLE_FMT_S16?
Here is a working example (in pAudioBuffer you have pcm data within white nose):
SwrContext *swr;
swr=swr_alloc();
av_opt_set_int(swr,"in_channel_layout",2,0);
av_opt_set_int(swr, "out_channel_layout", 2, 0);
av_opt_set_int(swr, "in_sample_rate", codecContext->sample_rate, 0);
av_opt_set_int(swr, "out_sample_rate", codecContext->sample_rate, 0);
av_opt_set_sample_fmt(swr, "in_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_S16P, 0);
swr_init(swr);
int16_t * pAudioBuffer = (int16_t *) av_malloc (AUDIO_INBUF_SIZE * 2);
while(av_read_frame(fmt_cntx,&readingPacket)==0){
if(readingPacket.stream_index==audioSteam->index){
AVPacket decodingPacket=readingPacket;
while(decodingPacket.size>0){
int gotFrame=0;
int result=avcodec_decode_audio4(codecContext,frame,&gotFrame,&decodingPacket);
if(result<0){
av_frame_free(&frame);
avformat_close_input(&fmt_cntx);
return null;
}
if(result>=0 && gotFrame){
int data_size=frame->nb_samples*frame->channels;
swr_convert(swr,&pAudioBuffer,frame->nb_samples,frame->extended_data,frame->nb_samples);
jshort *outShortArray=(*pEnv)->NewShortArray(pEnv,data_size);
(*pEnv)->SetShortArrayRegion(pEnv,outShortArray,0,data_size,pAudioBuffer);
(*pEnv)->CallVoidMethod(pEnv,pObj,callBackShortBuffer,outShortArray,data_size);
(*pEnv)->DeleteLocalRef(pEnv,outShortArray);
decodingPacket.size -= result;
decodingPacket.data += result;
}else{
decodingPacket.size=0;
decodingPacket.data=NULL;
}}
av_free_packet(&decodingPacket);
}

Related

FFMpeg How to use multithreading?

I want to decode H264 by ffmpeg, BUT finally I found the decode function only used one cpu core
system monitor
env: Ubuntu 14.04 FFMpeg 3.2.4 CPU i7-7500U
So, I search ffmpeg multithreading and decide using all cpu cores for decoding.
I set AVCodecContext as this:
//Init works
//codecId=AV_CODEC_ID_H264;
avcodec_register_all();
pCodec = avcodec_find_decoder(codecId);
if (!pCodec)
{
printf("Codec not found\n");
return -1;
}
pCodecCtx = avcodec_alloc_context3(pCodec);
if (!pCodecCtx)
{
printf("Could not allocate video codec context\n");
return -1;
}
pCodecParserCtx=av_parser_init(codecId);
if (!pCodecParserCtx)
{
printf("Could not allocate video parser context\n");
return -1;
}
pCodecCtx->thread_count = 4;
pCodecCtx->thread_type = FF_THREAD_FRAME;
pCodec->capabilities &= CODEC_CAP_TRUNCATED;
pCodecCtx->flags |= CODEC_FLAG_TRUNCATED;
if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
{
printf("Could not open codec\n");
return -1;
}
av_log_set_level(AV_LOG_QUIET);
av_init_packet(&packet);
//parse and decode
//after av_parser_parse2, the packet has a complete frame data
//in decode function, I just call avcodec_decode_video2 and do some frame copy work
while (cur_size>0)
{
int len = av_parser_parse2(
pCodecParserCtx, pCodecCtx,
&packet.data, &packet.size,
cur_ptr, cur_size,
AV_NOPTS_VALUE, AV_NOPTS_VALUE, AV_NOPTS_VALUE);
cur_ptr += len;
cur_size -= len;
if(GetPacketSize()==0)
continue;
AVFrame *pFrame = av_frame_alloc();
int ret = Decode(pFrame);
if (ret < 0)
{
continue;
}
if (ret)
{
//some works
}
}
But nothing different with before.
How can I use multithreading in FFMpeg? Any advise?
pCodec->capabilities &= CODEC_CAP_TRUNCATED;
And that's your bug. Please remove this line. The return value of avcodec_find_decoder() should for all practical intents and purposes be considered const.
Specifically, this statement removes the AV_CODEC_CAP_FRAME_THREADS flag from the codec's capabilities, thus effectively disabling frame-multithreading in the rest of the code.

C++ FFmpeg distorted sound when converting audio

I'm using the FFmpeg library to generate MP4 files containing audio from various files, such as MP3, WAV, OGG, but I'm having some troubles (I'm also putting video in there, but for simplicity's sake I'm omitting that for this question, since I've got that working). My current code opens an audio file, decodes the content and converts it into the MP4 container and finally writes it into the destination file as interleaved frames.
It works perfectly for most MP3 files, but when inputting WAV or OGG, the audio in the resulting MP4 is slightly distorted and often plays at the wrong speed (up to many times faster or slower).
I've looked at countless of examples of using the converting functions (swr_convert), but I can't seem to get rid of the noise in the exported audio.
Here's how I add an audio stream to the MP4 (outContext is the AVFormatContext for the output file):
audioCodec = avcodec_find_encoder(outContext->oformat->audio_codec);
if (!audioCodec)
die("Could not find audio encoder!");
// Start stream
audioStream = avformat_new_stream(outContext, audioCodec);
if (!audioStream)
die("Could not allocate audio stream!");
audioCodecContext = audioStream->codec;
audioStream->id = 1;
// Setup
audioCodecContext->sample_fmt = AV_SAMPLE_FMT_S16;
audioCodecContext->bit_rate = 128000;
audioCodecContext->sample_rate = 44100;
audioCodecContext->channels = 2;
audioCodecContext->channel_layout = AV_CH_LAYOUT_STEREO;
// Open the codec
if (avcodec_open2(audioCodecContext, audioCodec, NULL) < 0)
die("Could not open audio codec");
And to open a sound file from MP3/WAV/OGG (from the filename variable)...
// Create contex
formatContext = avformat_alloc_context();
if (avformat_open_input(&formatContext, filename, NULL, NULL)<0)
die("Could not open file");
// Find info
if (avformat_find_stream_info(formatContext, 0)<0)
die("Could not find file info");
av_dump_format(formatContext, 0, filename, false);
// Find audio stream
streamId = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
if (streamId < 0)
die("Could not find Audio Stream");
codecContext = formatContext->streams[streamId]->codec;
// Find decoder
codec = avcodec_find_decoder(codecContext->codec_id);
if (codec == NULL)
die("cannot find codec!");
// Open codec
if (avcodec_open2(codecContext, codec, 0)<0)
die("Codec cannot be found");
// Set up resample context
swrContext = swr_alloc();
if (!swrContext)
die("Failed to alloc swr context");
av_opt_set_int(swrContext, "in_channel_count", codecContext->channels, 0);
av_opt_set_int(swrContext, "in_channel_layout", codecContext->channel_layout, 0);
av_opt_set_int(swrContext, "in_sample_rate", codecContext->sample_rate, 0);
av_opt_set_sample_fmt(swrContext, "in_sample_fmt", codecContext->sample_fmt, 0);
av_opt_set_int(swrContext, "out_channel_count", audioCodecContext->channels, 0);
av_opt_set_int(swrContext, "out_channel_layout", audioCodecContext->channel_layout, 0);
av_opt_set_int(swrContext, "out_sample_rate", audioCodecContext->sample_rate, 0);
av_opt_set_sample_fmt(swrContext, "out_sample_fmt", audioCodecContext->sample_fmt, 0);
if (swr_init(swrContext))
die("Failed to init swr context");
Finally, to decode+convert+encode...
// Allocate and init re-usable frames
audioFrameDecoded = av_frame_alloc();
if (!audioFrameDecoded)
die("Could not allocate audio frame");
audioFrameDecoded->format = fileCodecContext->sample_fmt;
audioFrameDecoded->channel_layout = fileCodecContext->channel_layout;
audioFrameDecoded->channels = fileCodecContext->channels;
audioFrameDecoded->sample_rate = fileCodecContext->sample_rate;
audioFrameConverted = av_frame_alloc();
if (!audioFrameConverted)
die("Could not allocate audio frame");
audioFrameConverted->nb_samples = audioCodecContext->frame_size;
audioFrameConverted->format = audioCodecContext->sample_fmt;
audioFrameConverted->channel_layout = audioCodecContext->channel_layout;
audioFrameConverted->channels = audioCodecContext->channels;
audioFrameConverted->sample_rate = audioCodecContext->sample_rate;
AVPacket inPacket;
av_init_packet(&inPacket);
inPacket.data = NULL;
inPacket.size = 0;
int frameFinished = 0;
while (av_read_frame(formatContext, &inPacket) >= 0) {
if (inPacket.stream_index == streamId) {
int len = avcodec_decode_audio4(fileCodecContext, audioFrameDecoded, &frameFinished, &inPacket);
if (frameFinished) {
// Convert
uint8_t *convertedData=NULL;
if (av_samples_alloc(&convertedData,
NULL,
audioCodecContext->channels,
audioFrameConverted->nb_samples,
audioCodecContext->sample_fmt, 0) < 0)
die("Could not allocate samples");
int outSamples = swr_convert(swrContext,
&convertedData,
audioFrameConverted->nb_samples,
(const uint8_t **)audioFrameDecoded->data,
audioFrameDecoded->nb_samples);
if (outSamples < 0)
die("Could not convert");
size_t buffer_size = av_samples_get_buffer_size(NULL,
audioCodecContext->channels,
audioFrameConverted->nb_samples,
audioCodecContext->sample_fmt,
0);
if (buffer_size < 0)
die("Invalid buffer size");
if (avcodec_fill_audio_frame(audioFrameConverted,
audioCodecContext->channels,
audioCodecContext->sample_fmt,
convertedData,
buffer_size,
0) < 0)
die("Could not fill frame");
AVPacket outPacket;
av_init_packet(&outPacket);
outPacket.data = NULL;
outPacket.size = 0;
if (avcodec_encode_audio2(audioCodecContext, &outPacket, audioFrameConverted, &frameFinished) < 0)
die("Error encoding audio frame");
if (frameFinished) {
outPacket.stream_index = audioStream->index;
if (av_interleaved_write_frame(outContext, &outPacket) != 0)
die("Error while writing audio frame");
av_free_packet(&outPacket);
}
}
}
}
av_frame_free(&audioFrameConverted);
av_frame_free(&audioFrameDecoded);
av_free_packet(&inPacket);
I have also tried setting appropriate pts values for outgoing frames, but that doesn't seem to affect the sound quality at all.
I'm also unsure how/if I should be allocating the converted data, can av_samples_alloc be used for this? What about avcodec_fill_audio_frame? Am I on the right track?
Any input is appreciated (I can also send the exported MP4s if necessary, if you want to hear the distortion).
if (avcodec_encode_audio2(audioCodecContext, &outPacket, audioFrameConverted, &frameFinished) < 0)
die("Error encoding audio frame");
You seem to be assuming that the encoder will eat all submitted samples - it doesn't. It also doesn't cache them internally. It will eat a specific number of samples (AVCodecContext.frame_size), and the rest should be resubmitted in the next call to avcodec_encode_audio2().
[edit]
ok, so your edited code is better, but not there yet. You're still assuming the decoder will output at least frame_size samples for each call to avcodec_decode_audioN() (after resampling), which may not be the case. If that happens (and it does, for ogg), your avcodec_encode_audioN() call will encode an incomplete input buffer (because you say it's got frame_size samples, but it doesn't). Likewise, your code also doesn't deal with cases where the decoder outputs a number significantly bigger than frame_size (like 10*frame_size) expected by the encoder, in which case you'll get overruns - basically your 1:1 decode/encode mapping is the main source of your problem.
As a solution, consider the swrContext a FIFO, where you input all decoder samples, and loop over it until it's got less than frame_size samples left. I'll leave it up to you to learn how to deal with end-of-stream, because you'll need to flush cached samples out of the decoder (by calling avcodec_decode_audioN() with AVPacket where .data = NULL and .size = 0), flush the swrContext (by calling swr_context() until it returns 0) as well as flush the encoder (by feeding it NULL AVFrames until it returns AVPacket with .size = 0). Right now you'll probably get an output file where the end is slightly truncated. That shouldn't be hard to figure out.
This code works for me for m4a/ogg/mp3 to m4a/aac conversion:
#include "libswresample/swresample.h"
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavutil/opt.h"
#include <stdio.h>
#include <stdlib.h>
static void die(char *str) {
fprintf(stderr, "%s\n", str);
exit(1);
}
static AVStream *add_audio_stream(AVFormatContext *oc, enum AVCodecID codec_id)
{
AVCodecContext *c;
AVCodec *encoder = avcodec_find_encoder(codec_id);
AVStream *st = avformat_new_stream(oc, encoder);
if (!st) die("av_new_stream");
c = st->codec;
c->codec_id = codec_id;
c->codec_type = AVMEDIA_TYPE_AUDIO;
/* put sample parameters */
c->bit_rate = 64000;
c->sample_rate = 44100;
c->channels = 2;
c->sample_fmt = encoder->sample_fmts[0];
c->channel_layout = AV_CH_LAYOUT_STEREO;
// some formats want stream headers to be separate
if(oc->oformat->flags & AVFMT_GLOBALHEADER)
c->flags |= CODEC_FLAG_GLOBAL_HEADER;
return st;
}
static void open_audio(AVFormatContext *oc, AVStream *st)
{
AVCodecContext *c = st->codec;
AVCodec *codec;
/* find the audio encoder */
codec = avcodec_find_encoder(c->codec_id);
if (!codec) die("avcodec_find_encoder");
/* open it */
AVDictionary *dict = NULL;
av_dict_set(&dict, "strict", "+experimental", 0);
int res = avcodec_open2(c, codec, &dict);
if (res < 0) die("avcodec_open");
}
int main(int argc, char *argv[]) {
av_register_all();
if (argc != 3) {
fprintf(stderr, "%s <in> <out>\n", argv[0]);
exit(1);
}
// Allocate and init re-usable frames
AVCodecContext *fileCodecContext, *audioCodecContext;
AVFormatContext *formatContext, *outContext;
AVStream *audioStream;
SwrContext *swrContext;
int streamId;
// input file
const char *file = argv[1];
int res = avformat_open_input(&formatContext, file, NULL, NULL);
if (res != 0) die("avformat_open_input");
res = avformat_find_stream_info(formatContext, NULL);
if (res < 0) die("avformat_find_stream_info");
AVCodec *codec;
res = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0);
if (res < 0) die("av_find_best_stream");
streamId = res;
fileCodecContext = avcodec_alloc_context3(codec);
avcodec_copy_context(fileCodecContext, formatContext->streams[streamId]->codec);
res = avcodec_open2(fileCodecContext, codec, NULL);
if (res < 0) die("avcodec_open2");
// output file
const char *outfile = argv[2];
AVOutputFormat *fmt = fmt = av_guess_format(NULL, outfile, NULL);
if (!fmt) die("av_guess_format");
outContext = avformat_alloc_context();
outContext->oformat = fmt;
audioStream = add_audio_stream(outContext, fmt->audio_codec);
open_audio(outContext, audioStream);
res = avio_open2(&outContext->pb, outfile, AVIO_FLAG_WRITE, NULL, NULL);
if (res < 0) die("url_fopen");
avformat_write_header(outContext, NULL);
audioCodecContext = audioStream->codec;
// resampling
swrContext = swr_alloc();
av_opt_set_channel_layout(swrContext, "in_channel_layout", fileCodecContext->channel_layout, 0);
av_opt_set_channel_layout(swrContext, "out_channel_layout", audioCodecContext->channel_layout, 0);
av_opt_set_int(swrContext, "in_sample_rate", fileCodecContext->sample_rate, 0);
av_opt_set_int(swrContext, "out_sample_rate", audioCodecContext->sample_rate, 0);
av_opt_set_sample_fmt(swrContext, "in_sample_fmt", fileCodecContext->sample_fmt, 0);
av_opt_set_sample_fmt(swrContext, "out_sample_fmt", audioCodecContext->sample_fmt, 0);
res = swr_init(swrContext);
if (res < 0) die("swr_init");
AVFrame *audioFrameDecoded = av_frame_alloc();
if (!audioFrameDecoded)
die("Could not allocate audio frame");
audioFrameDecoded->format = fileCodecContext->sample_fmt;
audioFrameDecoded->channel_layout = fileCodecContext->channel_layout;
audioFrameDecoded->channels = fileCodecContext->channels;
audioFrameDecoded->sample_rate = fileCodecContext->sample_rate;
AVFrame *audioFrameConverted = av_frame_alloc();
if (!audioFrameConverted) die("Could not allocate audio frame");
audioFrameConverted->nb_samples = audioCodecContext->frame_size;
audioFrameConverted->format = audioCodecContext->sample_fmt;
audioFrameConverted->channel_layout = audioCodecContext->channel_layout;
audioFrameConverted->channels = audioCodecContext->channels;
audioFrameConverted->sample_rate = audioCodecContext->sample_rate;
AVPacket inPacket;
av_init_packet(&inPacket);
inPacket.data = NULL;
inPacket.size = 0;
int frameFinished = 0;
while (av_read_frame(formatContext, &inPacket) >= 0) {
if (inPacket.stream_index == streamId) {
int len = avcodec_decode_audio4(fileCodecContext, audioFrameDecoded, &frameFinished, &inPacket);
if (frameFinished) {
// Convert
uint8_t *convertedData=NULL;
if (av_samples_alloc(&convertedData,
NULL,
audioCodecContext->channels,
audioFrameConverted->nb_samples,
audioCodecContext->sample_fmt, 0) < 0)
die("Could not allocate samples");
int outSamples = swr_convert(swrContext, NULL, 0,
//&convertedData,
//audioFrameConverted->nb_samples,
(const uint8_t **)audioFrameDecoded->data,
audioFrameDecoded->nb_samples);
if (outSamples < 0) die("Could not convert");
for (;;) {
outSamples = swr_get_out_samples(swrContext, 0);
if (outSamples < audioCodecContext->frame_size * audioCodecContext->channels) break; // see comments, thanks to #dajuric for fixing this
outSamples = swr_convert(swrContext,
&convertedData,
audioFrameConverted->nb_samples, NULL, 0);
size_t buffer_size = av_samples_get_buffer_size(NULL,
audioCodecContext->channels,
audioFrameConverted->nb_samples,
audioCodecContext->sample_fmt,
0);
if (buffer_size < 0) die("Invalid buffer size");
if (avcodec_fill_audio_frame(audioFrameConverted,
audioCodecContext->channels,
audioCodecContext->sample_fmt,
convertedData,
buffer_size,
0) < 0)
die("Could not fill frame");
AVPacket outPacket;
av_init_packet(&outPacket);
outPacket.data = NULL;
outPacket.size = 0;
if (avcodec_encode_audio2(audioCodecContext, &outPacket, audioFrameConverted, &frameFinished) < 0)
die("Error encoding audio frame");
if (frameFinished) {
outPacket.stream_index = audioStream->index;
if (av_interleaved_write_frame(outContext, &outPacket) != 0)
die("Error while writing audio frame");
av_free_packet(&outPacket);
}
}
}
}
}
swr_close(swrContext);
swr_free(&swrContext);
av_frame_free(&audioFrameConverted);
av_frame_free(&audioFrameDecoded);
av_free_packet(&inPacket);
av_write_trailer(outContext);
avio_close(outContext->pb);
avcodec_close(fileCodecContext);
avcodec_free_context(&fileCodecContext);
avformat_close_input(&formatContext);
return 0;
}
I wanted to include a couple things I found when I was working with the above code.
I had one file get stuck in an infinite loop. The reason is the file had a sample rate of 48000 and the code changes it to a 44100. This caused it to always have extra outSamples. swr_convert & would not grab them. So I ended up changing add_audio_stream to match the input streams sample rate.
c->sample_rate = fileCodecContext->sample_rate;
Also I had to produce wav files as my output. And it had a framesize of 0. so I just chose a number after a few tests I went with 32. I noticed if I went too big (ex 128) I would get audio glitches.
if (audioFrameConverted->nb_samples <= 0) audioFrameConverted->nb_samples = 32; //wav files have a 0
Changed the if statement that breaks out of the loop to check nb_samples if frame_size is 0.
if ((outSamples < audioCodecContext->frame_size * audioCodecContext->channels) || audioCodecContext->frame_size==0 && (outSamples < audioFrameConverted->nb_samples * audioCodecContext->channels)) break; // see comments, thanks to #dajuric for fixing this
There was also a glitch when I was testing outputting to ogg files where the timestamp data was missing so the file wouldn't play correctly in vlc. There were a few lines I added that helped with that.
out_audioStream->time_base = in_audioStream->time_base; // entered before avio_open.
outPacket.dts = audioFrameDecoded->pkt_dts;//rest after avcodec_encode_audio2
outPacket.pts = audioFrameDecoded->pkt_pts;
av_packet_rescale_ts(&outPacket, in_audioStream->time_base, out_audioStream->time_base);
Variables might be a little different I converted the code to c#. Thought this might help someone.
Actually swr_convert won't work for that, try to use swr_convert_frame instead.

Encoding FLOAT PCM to OGG using libav

I am currently trying to convert a raw PCM Float buffer to an OGG encoded file. I tried several library to do the encoding process and I finally chose libavcodec.
What I precisely want to do is get the float buffer ([-1;1]) provided by my audio library and turn it to a char buffer of encoded ogg data.
I managed to encode the float buffer to a buffer of encoded MP2 with this (proof of concept) code:
static AVCodec *codec;
static AVCodecContext *c;
static AVPacket pkt;
static uint16_t* samples;
static AVFrame* frame;
static int frameEncoded;
FILE *file;
int main(int argc, char *argv[])
{
file = fopen("file.ogg", "w+");
long ret;
avcodec_register_all();
codec = avcodec_find_encoder(AV_CODEC_ID_MP2);
if (!codec) {
fprintf(stderr, "codec not found\n");
exit(1);
}
c = avcodec_alloc_context3(NULL);
c->bit_rate = 256000;
c->sample_rate = 44100;
c->channels = 2;
c->sample_fmt = AV_SAMPLE_FMT_S16;
c->channel_layout = AV_CH_LAYOUT_STEREO;
/* open it */
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
/* frame containing input raw audio */
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate audio frame\n");
exit(1);
}
frame->nb_samples = c->frame_size;
frame->format = c->sample_fmt;
frame->channel_layout = c->channel_layout;
/* the codec gives us the frame size, in samples,
* we calculate the size of the samples buffer in bytes */
int buffer_size = av_samples_get_buffer_size(NULL, c->channels, c->frame_size,
c->sample_fmt, 0);
if (buffer_size < 0) {
fprintf(stderr, "Could not get sample buffer size\n");
exit(1);
}
samples = av_malloc(buffer_size);
if (!samples) {
fprintf(stderr, "Could not allocate %d bytes for samples buffer\n",
buffer_size);
exit(1);
}
/* setup the data pointers in the AVFrame */
ret = avcodec_fill_audio_frame(frame, c->channels, c->sample_fmt,
(const uint8_t*)samples, buffer_size, 0);
if (ret < 0) {
fprintf(stderr, "Could not setup audio frame\n");
exit(1);
}
}
void myLibraryCallback(float *inbuffer, unsigned int length)
{
for(int j = 0; j < (2 * length); j++) {
if(frameEncoded >= (c->frame_size *2)) {
int avret, got_output;
av_init_packet(&pkt);
pkt.data = NULL; // packet data will be allocated by the encoder
pkt.size = 0;
avret = avcodec_encode_audio2(c, &pkt, frame, &got_output);
if (avret < 0) {
fprintf(stderr, "Error encoding audio frame\n");
exit(1);
}
if (got_output) {
fwrite(pkt.data, 1, pkt.size, file);
av_free_packet(&pkt);
}
frameEncoded = 0;
}
samples[frameEncoded] = inbuffer[j] * SHRT_MAX;
frameEncoded++;
}
}
The code is really simple, I initialize libavencode the usual way, then my audio library sends me processed PCM FLOAT [-1;1] interleaved at 44.1Khz and the number of floats (usually 1024) in the inbuffer for each channel (2 for stereo). So usually, inbuffer contains 2048 floats.
That was easy since I just needed here to convert my PCM to 16P, both interleaved. Moreover it is possible to code a 16P sample on a single char.
Now I would like to apply this to OGG which needs a sample format of AV_SAMPLE_FMT_FLTP.
Since my native format is AV_SAMPLE_FMT_FLT, it should only be some desinterleaving. Which is really easy to do.
The points I don't get are:
How can you send a float buffer on a char buffer ? Do we treat them as-is (float* floatSamples = (float*) samples) ? If so, what means the sample number avcodec gives you ? Is it the number of floats or chars ?
How can you send datas on two buffers (one for left, one for right) when avcodec_fill_audio_frame only takes a (uint8_t*) parameter and not a (uint8_t**) for multiple channels ? Does-it completely change the previous sample code ?
I tried to find some answers myself and I made a LOT of experiments so far but I failed on theses points. Since there is a huge lack of documentation on these, I would be very grateful if you had answers.
Thank you !

How do I convert ADPCM to PCM using FFmpeg?

I have a video feed that sends me audio using the ADPCM codec. However, android only supports PCM format. How can I convert the ADPCM audio feed into a PCM audio feed?
The answer to this may be similar to the answer to this question.
I have successfully decoded the frame with this code:
int len = avcodec_decode_audio4(pAudioCodecCtx, pAudioFrame, &frameFinished, &packet);
Is the secret here to use a reverse encode function?
Here is what I have so far in my audio decode function:
<!-- language: c -->
if(packet_queue_get(env, javaThread, pAudioPacketQueue, &packet, 1) < 0) {
LOGE("audio - after get packet failed");
return;
}
LOGD("Dequeued audio packet");
// calculate frame size
int frameSize;
if (pPcmAudioCodecCtx->frame_size) {
frameSize = pPcmAudioCodecCtx->frame_size;
} else {
/* if frame_size is not set, the number of samples must be
* calculated from the buffer size */
int64_t nb_samples = (int64_t)AUDIO_PCM_OUTBUFF_SIZE * 8 /
(av_get_bits_per_sample(pPcmAudioCodecCtx->codec_id) *
pPcmAudioCodecCtx->channels);
frameSize = nb_samples;
}
int pcmBytesPerSample = av_get_bytes_per_sample(pPcmAudioCodecCtx->sample_fmt);
int pcmFrameBytes = frameSize * pcmBytesPerSample * pPcmAudioCodecCtx->channels;
uint8_t *pDataStart = packet.data;
while(packet.size > 0) {
int len = avcodec_decode_audio4(pAudioCodecCtx, pAudioFrame, &frameFinished, &packet);
LOGD("Decoded ADPCM frame");
if (len < 0) {
LOGE("Error while decoding audio");
return;
}
if (frameFinished) {
// store frame data in FIFO buffer
uint8_t *inputBuffer = pAudioFrame->data[0];
int inputBufferSize = pAudioFrame->linesize[0];
av_fifo_generic_write(fifoBuffer, inputBuffer, inputBufferSize, NULL);
LOGD("Added ADPCM frame to FIFO buffer");
// check if fifo buffer has enough data for a PCM frame
while (av_fifo_size(fifoBuffer) >= pcmFrameBytes) {
LOGI("PCM frame data in FIFO buffer");
// read frame's worth of data from FIFO buffer
av_fifo_generic_read(fifoBuffer, pAudioPcmOutBuffer, pcmFrameBytes, NULL);
LOGD("Read data from FIFO buffer into pcm frame");
avcodec_get_frame_defaults(pPcmAudioFrame);
LOGD("Got frame defaults");
pPcmAudioFrame->nb_samples = pcmFrameBytes / (pPcmAudioCodecCtx->channels *
pcmBytesPerSample);
avcodec_fill_audio_frame(pPcmAudioFrame, pPcmAudioCodecCtx->channels,
pPcmAudioCodecCtx->sample_fmt,
pAudioPcmOutBuffer, pcmFrameBytes, 1);
LOGD("Filled frame audio with data");
// fill audio play buffer
int dataSize = pPcmAudioFrame->linesize[0];
LOGD("Data to output: %d", dataSize);
jbyteArray audioPlayBuffer = (jbyteArray) env->GetObjectField(ffmpegCtx, env->GetFieldID(cls, "audioPlayBuffer", "[B"));
jbyte *bytes = env->GetByteArrayElements(audioPlayBuffer, NULL);
memcpy(bytes, pPcmAudioFrame->data[0], dataSize);
env->ReleaseByteArrayElements(audioPlayBuffer, bytes, 0);
LOGD("Copied data into Java array");
env->CallVoidMethod(player, env->GetMethodID(playerCls, "updateAudio", "(I)V"), dataSize);
}
It turns out that the audio_decode_ functions return 16 bit PCM format, and that I just didn't know how to access it properly.
Here is the altered code inside the packet loop that plays the audio based on avcodec_decode_audio4.
int len = avcodec_decode_audio4(pAudioCodecCtx, pAudioFrame, &frameFinished, &packet);
if (len < 0) {
LOGE("Error while decoding audio");
return;
}
if (frameFinished) {
int planeSize;
uint8_t *pcmBuffer = pAudioFrame->extended_data[0];
int dataSize = av_samples_get_buffer_size(&planeSize, pAudioCodecCtx->channels,
pAudioFrame->nb_samples,
pAudioCodecCtx->sample_fmt, 1);
// fill audio play buffer
jbyteArray audioPlayBuffer = (jbyteArray) env->GetObjectField(ffmpegCtx, env->GetFieldID(cls, "audioPlayBuffer", "[B"));
jbyte *bytes = env->GetByteArrayElements(audioPlayBuffer, NULL);
memcpy(bytes, pcmBuffer, dataSize);
env->ReleaseByteArrayElements(audioPlayBuffer, bytes, 0);
env->CallVoidMethod(player, env->GetMethodID(playerCls, "updateAudio", "(I)V"), dataSize);
}
You can see sample code at http://ffmpeg.org/doxygen/trunk/doc_2examples_2decoding_encoding_8c-example.html
See audio_encode_example function.

encoding direcshow frame buffers by using libavcodec

I am trying to encode a stream buffer of frames grabbed by ISampleGrabber(directshow) by using libavcodec. After encoding those frame I am writing it into a file. But after completion file contains only green frames.
hers is code for grabbing frames and encoding it...
void DSGrabberCallback::initFFMpeg(){
const char* filename="G:/test1.mpg";
avcodec_register_all();
printf("Encode video file %s\n", filename);
AVCodecID codec_id=AV_CODEC_ID_MPEG2VIDEO;
codec = avcodec_find_encoder(codec_id);
c = avcodec_alloc_context3(codec);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
}
c->bit_rate = 4000000;
c->width = 320;
c->height = 240;
AVRational test;
test.den=25;
test.num=1;
c->time_base= test;
c->gop_size = 10;
//c->max_b_frames=1;
c->pix_fmt = AV_PIX_FMT_YUV420P;
if(codec_id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
}
f = fopen(filename, "wb");
if (!f) {
fprintf(stderr, "Could not open %s\n", filename);
}
picture = alloc_picture(c->pix_fmt, c->width, c->height);
/*picture->format = c->pix_fmt;
picture->width = c->width;
picture->height = c->height;*/
av_init_packet(&pkt);
}
void DSGrabberCallback::encodeFrame(unsigned char *frame,ULONG size){
std::cout<<"called.....";
pkt.data = NULL;
pkt.size = 0;
picture->data[0]=frame;
fflush(stdout);
picture->pts=counter;
ret = avcodec_encode_video2(c, &pkt, picture, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
}
if (got_output) {
printf("Write frame %3d (size=%5d)\n", counter, pkt.size);
fwrite(pkt.data, 1, pkt.size, f);
av_free_packet(&pkt);
}
}
STDMETHODIMP DSGrabberCallback::SampleCB(double time, IMediaSample* sample)
{
BYTE* data = NULL;
ULONG length = 0;
m_bytes=NULL;
counter=counter+1;
if(FAILED(sample->GetPointer(&data)))
{
return E_FAIL;
}
length = sample->GetActualDataLength();
if(length == 0)
{
return S_OK;
}
if(!m_bytes || m_bytesLength < length)
{
if(m_bytes)
{
delete[] m_bytes;
}
m_bytes = new unsigned char[length];
m_bytesLength = length;
}
if(true)
{
for(size_t row = 0 ; row < 480 ; row++)
{
memcpy((m_bytes + row * 640 * 2), data + (480 - 1 - row) * 640 * 2,
640 * 2);
}
}
std::cout<<"hiiiiiiiiiiiiiiiiiiiiiiii";
// memcpy(m_bytes, data, length);
// std::cout<<"called............... "<<m_bytes<<"\n";
if(counter<500){
encodeFrame(m_bytes,length);
}else{
fwrite(endcode, 1, sizeof(endcode), f);
fclose(f);
avcodec_close(c);
av_free(c);
av_freep(&picture->data[0]);
avcodec_free_frame(&picture);
printf("\n");
exit(1);
}
//rtp.sendRTP(data,length);
//sample->Release();
//printf("Sample received: %p %u\n", data, length);
return S_OK;
}
can anyone tell me where is the problem.
Now working fine. Actually I forgot to convert the image buffer into YUV420P format. I have added some code for scaling the buffer into YUV format and everything is fine now. Thank you Wimmel and Roman R.

Resources