Direct-X 12 is using the wrong and multiple adapters in C++ - graphics

I'm coding a C++ game from scratch using the Direct-X 12 API.
I'm using a few tutorials: Braynzar Soft Direct-X 12 tutorial, Alian Direct-X 12 and a few others.
Each one I use runs and has no errors. But I have a slight problem.
In professional games and game engines, they can use a dedicated GPU to run their programs and sometimes at a low performance. BUT, when I use this tutorials or I code it myself, it uses both of my GPUs simultaneously. Why is that? Is there some bug within the code? Does Direct-X 12 just multi-GPU automatically? Can I prevent this and how? There wouldn't be a problem with this, but I would like, if possible, to only use on GPU.
Simple code:
int32_t CALLBACK wWinMain(_In_ HINSTANCE, _In_opt_ HINSTANCE, _In_ LPWSTR, _In_ int32_t)
{
HRESULT ReturnCode = S_OK;
#ifdef _SCPDebug
ID3D12Debug3* D3D12DebugInterface = nullptr;
IDXGIInfoQueue* DXGIDebugInfoQueue = nullptr;
ReturnCode = D3D12GetDebugInterface(IID_PPV_ARGS(&D3D12DebugInterface));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
D3D12DebugInterface->EnableDebugLayer();
D3D12DebugInterface->SetEnableGPUBasedValidation(TRUE);
ReturnCode = DXGIGetDebugInterface1(0, IID_PPV_ARGS(&DXGIDebugInfoQueue));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = DXGIDebugInfoQueue->SetBreakOnSeverity(DXGI_DEBUG_ALL, DXGI_INFO_QUEUE_MESSAGE_SEVERITY_ERROR, TRUE);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = DXGIDebugInfoQueue->SetBreakOnSeverity(DXGI_DEBUG_ALL, DXGI_INFO_QUEUE_MESSAGE_SEVERITY_CORRUPTION, TRUE);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
#endif
IDXGIFactory7* DXGIFactory = nullptr;
#ifdef _SCPDebug
ReturnCode = CreateDXGIFactory2(1, IID_PPV_ARGS(&DXGIFactory));
#else
ReturnCode = CreateDXGIFactory2(0, IID_PPV_ARGS(&DXGIFactory));
#endif
if (ReturnCode != S_OK)
{
return ReturnCode;
}
BOOL SupportsVariableRefreshRate = FALSE;
ReturnCode = DXGIFactory->CheckFeatureSupport(DXGI_FEATURE_PRESENT_ALLOW_TEARING, &SupportsVariableRefreshRate, sizeof(BOOL));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
IDXGIAdapter4* DXGIAdapter = nullptr;
ReturnCode = DXGIFactory->EnumAdapterByGpuPreference(0, DXGI_GPU_PREFERENCE_HIGH_PERFORMANCE, IID_PPV_ARGS(&DXGIAdapter));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ID3D12Device8* D3D12Device = nullptr;
#ifdef _SCPDebug
ID3D12InfoQueue* D3D12DeviceDebugInfoQueue = nullptr;
#endif
ReturnCode = D3D12CreateDevice(DXGIAdapter, D3D_FEATURE_LEVEL_12_1, IID_PPV_ARGS(&D3D12Device));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12Device->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Device8");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
#ifdef _SCPDebug
ReturnCode = D3D12Device->QueryInterface(&D3D12DeviceDebugInfoQueue);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12DeviceDebugInfoQueue->SetBreakOnSeverity(D3D12_MESSAGE_SEVERITY_CORRUPTION, TRUE);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12DeviceDebugInfoQueue->SetBreakOnSeverity(D3D12_MESSAGE_SEVERITY_ERROR, TRUE);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
#endif
ID3D12DescriptorHeap* D3D12DSDescriptorHeap = nullptr;
ID3D12Resource2* D3D12DS = nullptr;
D3D12_DESCRIPTOR_HEAP_DESC DescriptorHeapDesc = {};
DescriptorHeapDesc.NumDescriptors = 1;
DescriptorHeapDesc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_DSV;
DescriptorHeapDesc.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_NONE;
ReturnCode = D3D12Device->CreateDescriptorHeap(&DescriptorHeapDesc, IID_PPV_ARGS(&D3D12DSDescriptorHeap));
if (FAILED(ReturnCode))
{
return ReturnCode;
}
ReturnCode = D3D12DSDescriptorHeap->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Depth Stencil Descriptor Heap");
if (FAILED(ReturnCode))
{
return ReturnCode;
}
D3D12_CLEAR_VALUE OptimizedDepthClearValue;
OptimizedDepthClearValue.DepthStencil.Depth = 1.0f;
OptimizedDepthClearValue.DepthStencil.Stencil = 0;
OptimizedDepthClearValue.Format = DXGI_FORMAT_D32_FLOAT;
D3D12_HEAP_PROPERTIES HeapProperties = CD3DX12_HEAP_PROPERTIES(D3D12_HEAP_TYPE_DEFAULT);
D3D12_RESOURCE_DESC HeapDesc = CD3DX12_RESOURCE_DESC::Tex2D(DXGI_FORMAT_D32_FLOAT, 1920, 1080, 1, 1, 1, 0, D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL);
ReturnCode = D3D12Device->CreateCommittedResource(&HeapProperties, D3D12_HEAP_FLAG_NONE, &HeapDesc, D3D12_RESOURCE_STATE_DEPTH_WRITE, &OptimizedDepthClearValue, IID_PPV_ARGS(D3D12DS));
if (FAILED(ReturnCode))
{
return ReturnCode;
}
ReturnCode = D3D12DS->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Depth Stencil Heap");
if (FAILED(ReturnCode))
{
return ReturnCode;
}
D3D12_DEPTH_STENCIL_VIEW_DESC DepthStencilViewDesc = {};
DepthStencilViewDesc.Format = DXGI_FORMAT_D32_FLOAT;
DepthStencilViewDesc.ViewDimension = D3D12_DSV_DIMENSION_TEXTURE2D;
DepthStencilViewDesc.Flags = D3D12_DSV_FLAG_NONE;
D3D12Device->CreateDepthStencilView(D3D12DS, &DepthStencilViewDesc, D3D12DSDescriptorHeap->GetCPUDescriptorHandleForHeapStart());
ID3D12Fence1* D3D12Fence = nullptr;
uint64_t FenceValue = 0ULL;
HANDLE FenceEvent = nullptr;
ReturnCode = D3D12Device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&D3D12Fence));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12Fence->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Sync::Fence");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
HANDLE FenceEvent = CreateEventW(nullptr, FALSE, FALSE, L"SCP-Site Breach::Engine::Direct-X 12::Sync::Fence Event");
if (FenceEvent == nullptr)
{
return E_UNEXPECTED;
}
ID3D12RootSignature* D3D12RootSignature = nullptr;
{
D3D12_ROOT_SIGNATURE_DESC1 Desc = {};
Desc.Flags = D3D12_ROOT_SIGNATURE_FLAG_ALLOW_INPUT_ASSEMBLER_INPUT_LAYOUT | D3D12_ROOT_SIGNATURE_FLAG_DENY_VERTEX_SHADER_ROOT_ACCESS |
D3D12_ROOT_SIGNATURE_FLAG_DENY_HULL_SHADER_ROOT_ACCESS | D3D12_ROOT_SIGNATURE_FLAG_DENY_DOMAIN_SHADER_ROOT_ACCESS |
D3D12_ROOT_SIGNATURE_FLAG_DENY_GEOMETRY_SHADER_ROOT_ACCESS | D3D12_ROOT_SIGNATURE_FLAG_DENY_PIXEL_SHADER_ROOT_ACCESS;
Desc.NumParameters = 0;
Desc.NumStaticSamplers = 0;
Desc.pParameters = nullptr;
Desc.pStaticSamplers = nullptr;
D3D12_VERSIONED_ROOT_SIGNATURE_DESC RootSignatureDescVersion = {};
RootSignatureDescVersion.Desc_1_1 = Desc;
RootSignatureDescVersion.Version = D3D_ROOT_SIGNATURE_VERSION_1_1;
ID3D10Blob* D3D12SignatureBlob = nullptr;
ID3D10Blob* D3D12ErrorBlob = nullptr;
ReturnCode = D3D12SerializeVersionedRootSignature(&RootSignatureDescVersion, &D3D12SignatureBlob &D3D12ErrorBlob);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12Device->CreateRootSignature(0, D3D12SignatureBlob->GetBufferPointer(), D3D12SignatureBlob->GetBufferSize(), IID_PPV_ARGS(&D3D12RootSignature));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode =D3D12RootSignature->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Root Signature");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
D3D12SignatureBlob->Release();
D3D12SignatureBlob = nullptr;
}
D3D12_SHADER_BYTECODE D3D12PixelShader = {};
D3D12_SHADER_BYTECODE D3D12VertexShader = {};
std::ifstream PixelShader("PixelShader.cso", std::ios_base::binary | std::ios_base::in);
auto FileSize = PixelShader.tellg();
PixelShader.seekg(0, std::iostream::end);
FileSize += PixelShader.tellg() - FileSize;
PixelShader.seekg(0, std::iostream::beg);
std::ifstream VertexShader("VertexShader.cso", std::ios_base::binary | std::ios_base::in);
auto FileSize2 = VertexShader.tellg();
VertexShader.seekg(0, std::iostream::end);
FileSize2 += VertexShader.tellg() - FileSize2;
VertexShader.seekg(0, std::iostream::beg);
D3D12PixelShader.BytecodeLength = msl::utilities::SafeInt<SIZE_T>(FileSize.operator long long());
D3D12PixelShader.pShaderBytecode = malloc(D3D12PixelShader.BytecodeLength);
PixelShader.read(reinterpret_cast<char*>(const_cast<void*>(D3D12PixelShader.pShaderBytecode)), msl::utilities::SafeInt<long long>(D3D12PixelShader.BytecodeLength));
D3D12VertexShader.BytecodeLength = msl::utilities::SafeInt<SIZE_T>(FileSize2.operator long long());
D3D12VertexShader.pShaderBytecode = malloc(D3D12VertexShader.BytecodeLength);
VertexShader.read(reinterpret_cast<char*>(const_cast<void*>(D3D12VertexShader.pShaderBytecode)), msl::utilities::SafeInt<long long>(D3D12VertexShader.BytecodeLength));
ID3D12PipelineState* D3D12PipelineStateObject = nullptr;
static const D3D12_INPUT_ELEMENT_DESC s_inputElementDesc[2] =
{
{ "SV_Position", 0, DXGI_FORMAT_R32G32B32A32_FLOAT, 0, 0, D3D12_INPUT_CLASSIFICATION_PER_VERTEX_DATA, 0 },
{ "COLOR", 0, DXGI_FORMAT_R32G32B32A32_FLOAT, 0, 16, D3D12_INPUT_CLASSIFICATION_PER_VERTEX_DATA , 0 }
};
D3D12_GRAPHICS_PIPELINE_STATE_DESC psoDesc = {};
psoDesc.InputLayout = { &s_inputElementDesc[0], _countof(s_inputElementDesc)};
psoDesc.pRootSignature = D3D12RootSignature;
psoDesc.PS = D3D12PixelShader;
psoDesc.VS = D3D12VertexShader;
psoDesc.RasterizerState = CD3DX12_RASTERIZER_DESC(D3D12_DEFAULT);
psoDesc.BlendState = CD3DX12_BLEND_DESC(D3D12_DEFAULT);
psoDesc.DepthStencilState = CD3DX12_DEPTH_STENCIL_DESC(D3D12_DEFAULT);
psoDesc.DSVFormat = DXGI_FORMAT_B8G8R8A8_UNORM_SRGB;
psoDesc.SampleMask = UINT_MAX;
psoDesc.PrimitiveTopologyType = D3D12_PRIMITIVE_TOPOLOGY_TYPE_TRIANGLE;
psoDesc.NumRenderTargets = 1;
psoDesc.RTVFormats[0] = DXGI_FORMAT_B8G8R8A8_UNORM_SRGB;
psoDesc.SampleDesc.Count = 1;
ReturnCode = D3D12Device->CreateGraphicsPipelineState(&psoDesc, IID_PPV_ARGS(&D3D12PipelineStateObject));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12PipelineStateObject->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Pipeline State");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ID3D12CommandQueue* D3D12CommandQueue = nullptr;
D3D12_COMMAND_QUEUE_DESC Desc = {};
Desc.Flags = D3D12_COMMAND_QUEUE_FLAG_NONE;
Desc.NodeMask = 0;
Desc.Priority = 0;
Desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
ReturnCode = D3D12Device->CreateCommandQueue(&Desc, IID_PPV_ARGS(&D3D12CommandQueue));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12CommandQueue->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Commands::Queue");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ID3D12GraphicsCommandList6* D3D12CommandList = nullptr;
ID3D12CommandAllocator* D3D12CommandAllocator = nullptr;
ReturnCode = D3D12Device->CreateCommandAllocator(_Type, IID_PPV_ARGS(&D3D12CommandAllocator));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12CommandAllocator->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Commands::Allocator");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12Device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, D3D12CommandAllocator, D3D12PipelineStateObject, IID_PPV_ARGS(&D3D12CommandList));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12CommandList->SetName(L"SCP-Site Breach::Engine::Direct-X 12::Commands::List");
if (ReturnCode != S_OK)
{
return ReturnCode;
}
ReturnCode = D3D12CommandList->Close();
if (ReturnCode != S_OK)
{
return ReturnCode;
}
IDXGISwapChain4* DXGISwapChain = nullptr;
ID3D12DescriptorHeap* D3D12SwapChainDescriptorHeap = nullptr;
std::vector<ID3D12Resource2*> D3D12SwapChainBuffers = { nullptr, nullptr };
WNDCLASSEXW WndClass = {};
WndClass.cbSize = sizeof(WNDCLASSEXW);
WndClass.hIcon = nullptr;
WndClass.hIconSm = nullptr;
WndClass.hCursor = LoadCursorW(nullptr, IDC_ARROW);
WndClass.hInstance = GetModuleHandleW(nullptr);
WndClass.lpfnWndProc = WindowProc;
WndClass.lpszClassName = g_WndClassName.c_str();
WndClass.hbrBackground = CreateSolidBrush(RGB(0, 0, 0));
if (!RegisterClassExW(&WndClass))
{
return -1;
}
HWND m_GameWindow = CreateWindowExW(0, g_WndClassName.c_str(), L"Robo_Lab", WS_OVERLAPPED, 0, 0, 500, 500, nullptr, nullptr, GetModuleHandleW(nullptr), nullptr);
SetWindowPos(m_GameWindow, HWND_TOP, 0, 0, 1920, 1080, SWP_SHOWWINDOW);
DXGI_SWAP_CHAIN_DESC SwapChainDesc = {};
SwapChainDesc.BufferCount = 2;
SwapChainDesc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
SwapChainDesc.BufferDesc.Height = 1080;
SwapChainDesc.BufferDesc.RefreshRate.Denominator = 1;
SwapChainDesc.BufferDesc.RefreshRate.Numerator = 60;
SwapChainDesc.BufferDesc.Scaling = DXGI_MODE_SCALING_STRETCHED;
SwapChainDesc.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_PROGRESSIVE;
SwapChainDesc.BufferDesc.Width = 1920;
SwapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
SwapChainDesc.OutputWindow = m_GameWindow;
SwapChainDesc.SampleDesc.Count = 1;
SwapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
SwapChainDesc.Windowed = true;
ReturnCode = DXGIFactory->CreateSwapChain(D3D12CommandQueue, &SwapChainDesc, reinterpret_cast<IDXGISwapChain**>(&DXGISwapChain));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
D3D12_DESCRIPTOR_HEAP_DESC DescriptorHeapDesc = {};
DescriptorHeapDesc.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_NONE;
DescriptorHeapDesc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_RTV;
DescriptorHeapDesc.NumDescriptors = 2;
ReturnCode = D3D12Device->CreateDescriptorHeap(&DescriptorHeapDesc, IID_PPV_ARGS(&D3D12SwapChainDescriptorHeap));
if (ReturnCode != S_OK)
{
return ReturnCode;
}
uint32_t RTVDescriptorSize = D3D12Device->GetDescriptorHandleIncrementSize(D3D12_DESCRIPTOR_HEAP_TYPE_RTV);
CD3DX12_CPU_DESCRIPTOR_HANDLE RTVDescriptorHandle(&D3D12SwapChainDescriptorHeap->GetCPUDescriptorHandleForHeapStart());
for (size_t i = 0; i < 2; ++i)
{
DXGISwapChain->GetBuffer(i, IID_PPV_ARGS(&D3D12SwapChainBuffers.at(i)));
wchar_t BufferName[64] = { 0 };
swprintf_s(BufferName, L"SCP-Site Breach::Engine::Direct-X 12::Swap Chain Buffer[%i]", i);
ReturnCode = D3D12SwapChainBuffers.at(i)->SetName(BufferName);
if (ReturnCode != S_OK)
{
return ReturnCode;
}
D3D12Device->CreateRenderTargetView(D3D12SwapChainBuffers.at(i), nullptr, RTVDescriptorHandle);
RTVDescriptorHandle.Offset(1, RTVDescriptorSize);
}
MSG WndMessage = { nullptr };
IsRunning = true;
D3D12_RESOURCE_BARRIER BackBufferResourceBarrier = {};
BackBufferResourceBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
BackBufferResourceBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;
BackBufferResourceBarrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
const float clearColor[] = { 0.0f, 0.2f, 0.4f, 1.0f };
CD3DX12_CPU_DESCRIPTOR_HANDLE dsvHandle(D3D12DSDescriptorHeap->GetCPUDescriptorHandleForHeapStart());
while (IsRunning)
{
if (PeekMessageW(&WndMessage, m_GameWindow, 0, 0, PM_REMOVE))
{
TranslateMessage(&WndMessage);
DispatchMessageW(&WndMessage);
}
if (GetWindow(m_GameWindow, 0))
{
if (D3D12Fence->GetCompletedValue() < FenceValue)
{
D3D12Fence->SetEventOnCompletion(FenceValue, FenceEvent);
WaitForSingleObject(FenceEvent, INFINITE);
}
CommandList.GetCommandAllocator()->Reset();
D3D12CommandList->Reset(CommandList.GetCommandAllocator(), Pipeline.GetPipeline());
FenceValue++;
BackBufferResourceBarrier.Transition.pResource = D3D12SwapChainBuffers.at(DXGISwapChain->GetCurrentBackBufferIndex());
BackBufferResourceBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_PRESENT;
BackBufferResourceBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_RENDER_TARGET;
D3D12CommandList->ResourceBarrier(1, &BackBufferResourceBarrier);
CD3DX12_CPU_DESCRIPTOR_HANDLE rtvHandle(D3D12SwapChainDescriptorHeap->GetCPUDescriptorHandleForHeapStart(), D3D12SwapChainBuffers.at(DXGISwapChain->GetCurrentBackBufferIndex()), D3D12Device->GetDescriptorHandleIncrementSize(D3D12_DESCRIPTOR_HEAP_TYPE_RTV));
D3D12CommandList->OMSetRenderTargets(1, &rtvHandle, FALSE, &dsvHandle);
D3D12CommandList->ClearRenderTargetView(rtvHandle, clearColor, 0, nullptr);
D3D12CommandList->ClearDepthStencilView(D3D12DSDescriptorHeap->GetCPUDescriptorHandleForHeapStart(), D3D12_CLEAR_FLAG_DEPTH, 1.0f, 0, 0, nullptr);
D3D12CommandList->SetGraphicsRootSignature(D3D12CommandQueue);
BackBufferResourceBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET;
BackBufferResourceBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_PRESENT;
D3D12CommandList->ResourceBarrier(1, &BackBufferResourceBarrier);
D3D12CommandList->Close();
D3D12CommandQueue->ExecuteCommandLists(1, reinterpret_cast<ID3D12CommandList**>(&D3D12CommandList));
D3D12CommandQueue->Signal(D3D12Fence, FenceValue);
DXGISwapChain->Present(0, 0);
}
}
DXGISwapChain->SetFullscreenState(FALSE, nullptr);
DXGISwapChain->Release();
D3D12SwapChainDescriptorHeap->Release();
D3D12SwapChainBuffers.at(0)->Release();
D3D12SwapChainBuffers.at(1)->Release();
D3D12CommandList->Release();
D3D12CommandAllocator->Release()
D3D12CommandQueue->Release();
D3D12PipelineStateObject->Release();
D3D12CommandQueue->Release();
CloseHandle(FenceEvent);
D3D12Fence->Release();
D3D12DSDescriptorHeap->Release();
D3D12DS->Release();
D3D12Device->Release();
#ifdef _SCPDebug
D3D12DeviceDebugInfoQueue->Release();
#endif
DXGIAdapter->Release();
DXGIFactory->Release();
D3D12DebugInterface->Release();
DXGIDebugInfoQueue->Release();
return 0;
}
Note: I converted this code from it's orginal format to this on keeping all of the settings and code. If something got messed up in the procress or converting it, it was a convertion error because this code worked fine except for the fact it was using both of my GPUs.
Edit:
I forgot to mention which GPUs it was using. I have two GPUs. One is a GTX 1650, which is the one I want to use for all of my games, then I have a Intel UHD Graphics 630 (which is the GPU it wants to use) and it's my display driver. When I get the adapter and I get the description, it says it chooses my GTX, but when I look at task manager, it says it's using my display driver's 3D, and my GTX's 3D and copy. Does it have to do when I use "IDXGISwapChain::Present"?

You might have misunderstood what the 3D queue of your integrated GPU is doing.
It's not doing the rendering work, but it is handling the entire desktop composition workload. Which GPU that happens on is determined by where your monitor's framebuffer is located on.
The Taskmanager is misleading as far as you are concerned. It shows you which engine on the GPU side is active (and it shows active time only, no load indicator!), but not from which software side queue this workload was submitted. If you take a look with GPUView instead, you will most likely see it's workload submitted by WDM, not by your process. All usage percentages you see are also wrong, as they are not adjusted for dynamic clock speeds. Your iGPU might as well be in low-power-mode and still show an 80% utilization.
Just how much the internal GPU has to work depends on your swapchain configuration. All the windowed modes naturally result in a significantly higher overhead than the full screen modes (that includes borderless full-screen!), which shows in utilization.

Related

Does wasapi support device which has 8 audio channels?

I tried to use wasapi to record system's playout audio, but if the audio device has 8 channels, I always get the error (AUDCLNT_E_UNSUPPORTED_FORMAT)
"m_captureClient->Initialize" would return AUDCLNT_E_UNSUPPORTED_FORMAT.
But if the device is 48khz, 2 channels, then everything is OK.
The code is as below:
do {
HRESULT hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
__uuidof(IMMDeviceEnumerator), (void**)&m_deviceEnumerator);
if (FAILED(hr)) {
err = ERROR_CODE_COM_CREATE_INSTANCE_FAILED;
break;
}
hr = m_deviceEnumerator->GetDevice(HELPER::StringConverter::convertUtf8ToUnicode(m_deviceId).c_str(), &m_endpointDevice);
if (FAILED(hr)) {
err = ERROR_CODE_COM_GET_AUDIO_ENDPOINT_FAILED;
break;
}
hr = m_endpointDevice->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, (void**)&m_captureClient);
if (FAILED(hr)) {
err = ERROR_CODE_COM_ACTIVE_DEVICE_FAILED;
break;
}
hr = m_captureClient->GetMixFormat(&m_waveFmt);
if (FAILED(hr)) {
err = ERROR_CODE_COM_GET_MIX_FORMAT_FAILED;
break;
}
initFromMixFormat(m_waveFmt);
DWORD streamFlags = AUDCLNT_STREAMFLAGS_EVENTCALLBACK;
if (!m_isMic) {
streamFlags |= AUDCLNT_STREAMFLAGS_LOOPBACK;
}
hr = m_captureClient->Initialize(AUDCLNT_SHAREMODE_SHARED, streamFlags, WASAPI_CAPTOR_REF_TIME_PER_SEC, 0, m_waveFmt, nullptr);
if (FAILED(hr)) {
err = ERROR_CODE_COM_INIT_AUDIO_CLIENT_FAILED;
break;
}
// For ouotput mode, capture event will not signal when there is nothing rendering,
// so we run a render thread and rendering silent pcm data all the time.
if (!m_isMic) {
err = initRenderer();
HCMDR_ERROR_CODE_BREAK(err);
}
hr = m_captureClient->GetBufferSize(&m_captureSampleCount);
if (FAILED(hr)) {
err = ERROR_CODE_COM_GET_BUFFER_SIZE_FAILED;
break;
}
hr = m_captureClient->GetService(__uuidof(IAudioCaptureClient), (void**)&m_captor);
if (FAILED(hr)) {
err = ERROR_CODE_COM_GET_CAPTOR_FAILED;
break;
}
m_captureEvent = CreateEventA(nullptr, FALSE, FALSE, nullptr);
if (m_captureEvent == nullptr) {
err = ERROR_CODE_COM_CREATE_EVENT_FAILED;
break;
}
m_stopEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr);
if (m_stopEvent == nullptr) {
err = ERROR_CODE_COM_CREATE_EVENT_FAILED;
break;
}
hr = m_captureClient->SetEventHandle(m_captureEvent);
if (FAILED(hr)) {
err = ERROR_CODE_COM_SET_EVENT_FAILED;
break;
}
m_inited = true;
} while (0);

ffmpeg record video plays too fast

I'm a college student and I am studying FFmpeg now.
I have wrote a software that can record desktops and audio('virtual-audio-capturer') with FFmpeg.And I am now writing Audio and Video Synchronization.
I met some problems that video recording plays too fast.
When I look for audio and video synchronization help on the Internet,I find a formula for calculating PTS :
pts = n * ((1 / timbase)/ fps)
When I use this formula,I find a phenomenon.
1.The higher frame rate is,the faster the video playback speed.
2.The slower the frame rate, the faster the video playback.
Also I find while the framerate is 10,the video playback speed will be right.
Why has this situation happened?
I have thought this question for three days. I really hope someone can help me solve this problem.
I really appreciate the help.
#include "stdafx.h"
#ifdef __cplusplus
extern "C"
{
#endif
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavdevice/avdevice.h"
#include "libavutil/audio_fifo.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavutil/imgutils.h"
#include "libavutil/mathematics.h"
#include "libavutil/samplefmt.h"
#include "libavutil/time.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
#include "libavutil/file.h"
#include "libavutil/mem.h"
#include "libavutil/frame.h"
#include "libavfilter/avfilter.h"
#include "libswresample/swresample.h"
#pragma comment(lib, "avcodec.lib")
#pragma comment(lib, "avformat.lib")
#pragma comment(lib, "avutil.lib")
#pragma comment(lib, "avdevice.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "postproc.lib")
#pragma comment(lib, "swresample.lib")
#pragma comment(lib, "swscale.lib")
#ifdef __cplusplus
};
#endif
AVFormatContext *pFormatCtx_Video = NULL, *pFormatCtx_Audio = NULL, *pFormatCtx_Out = NULL;
AVCodecContext *outVideoCodecCtx = NULL;
AVCodecContext *outAudioCodecCtx = NULL;
AVStream *pVideoStream = NULL, *pAudioStream = NULL;
AVCodec *outAVCodec;
AVCodec *outAudioCodec;
AVCodecContext *pCodecCtx_Video;
AVCodec *pCodec_Video;
AVFifoBuffer *fifo_video = NULL;
AVAudioFifo *fifo_audio = NULL;
int VideoIndex, AudioIndex;
int codec_id;
CRITICAL_SECTION AudioSection, VideoSection;
SwsContext *img_convert_ctx;
int frame_size = 0;
uint8_t *picture_buf = NULL, *frame_buf = NULL;
bool bCap = true;
DWORD WINAPI ScreenCapThreadProc( LPVOID lpParam );
DWORD WINAPI AudioCapThreadProc( LPVOID lpParam );
int OpenVideoCapture()
{
AVInputFormat *ifmt=av_find_input_format("gdigrab");
AVDictionary *options = NULL;
av_dict_set(&options, "framerate", "60", NULL);
if(avformat_open_input(&pFormatCtx_Video, "desktop", ifmt, &options)!=0)
{
printf("Couldn't open input stream.(无法打开视频输入流)\n");
return -1;
}
if(avformat_find_stream_info(pFormatCtx_Video,NULL)<0)
{
printf("Couldn't find stream information.(无法获取视频流信息)\n");
return -1;
}
if (pFormatCtx_Video->streams[0]->codec->codec_type != AVMEDIA_TYPE_VIDEO)
{
printf("Couldn't find video stream information.(无法获取视频流信息)\n");
return -1;
}
pCodecCtx_Video = pFormatCtx_Video->streams[0]->codec;
pCodec_Video = avcodec_find_decoder(pCodecCtx_Video->codec_id);
if(pCodec_Video == NULL)
{
printf("Codec not found.(没有找到解码器)\n");
return -1;
}
if(avcodec_open2(pCodecCtx_Video, pCodec_Video, NULL) < 0)
{
printf("Could not open codec.(无法打开解码器)\n");
return -1;
}
av_dump_format(pFormatCtx_Video, 0, NULL, 0);
img_convert_ctx = sws_getContext(pCodecCtx_Video->width, pCodecCtx_Video->height, pCodecCtx_Video->pix_fmt,
pCodecCtx_Video->width, pCodecCtx_Video->height, PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL);
frame_size = avpicture_get_size(pCodecCtx_Video->pix_fmt, pCodecCtx_Video->width, pCodecCtx_Video->height);
fifo_video = av_fifo_alloc(30 * avpicture_get_size(AV_PIX_FMT_YUV420P, pCodecCtx_Video->width, pCodecCtx_Video->height));
return 0;
}
static char *dup_wchar_to_utf8(wchar_t *w)
{
char *s = NULL;
int l = WideCharToMultiByte(CP_UTF8, 0, w, -1, 0, 0, 0, 0);
s = (char *) av_malloc(l);
if (s)
WideCharToMultiByte(CP_UTF8, 0, w, -1, s, l, 0, 0);
return s;
}
int OpenAudioCapture()
{
AVInputFormat *pAudioInputFmt = av_find_input_format("dshow");
char * psDevName = dup_wchar_to_utf8(L"audio=virtual-audio-capturer");
if (avformat_open_input(&pFormatCtx_Audio, psDevName, pAudioInputFmt,NULL) < 0)
{
printf("Couldn't open input stream.(无法打开音频输入流)\n");
return -1;
}
if(avformat_find_stream_info(pFormatCtx_Audio,NULL)<0)
return -1;
if(pFormatCtx_Audio->streams[0]->codec->codec_type != AVMEDIA_TYPE_AUDIO)
{
printf("Couldn't find video stream information.(无法获取音频流信息)\n");
return -1;
}
AVCodec *tmpCodec = avcodec_find_decoder(pFormatCtx_Audio->streams[0]->codec->codec_id);
if(0 > avcodec_open2(pFormatCtx_Audio->streams[0]->codec, tmpCodec, NULL))
{
printf("can not find or open audio decoder!\n");
}
av_dump_format(pFormatCtx_Audio, 0, NULL, 0);
return 0;
}
int OpenOutPut()
{
AVStream *pVideoStream = NULL, *pAudioStream = NULL;
const char *outFileName = "test.mp4";
avformat_alloc_output_context2(&pFormatCtx_Out, NULL, NULL, outFileName);
if (pFormatCtx_Video->streams[0]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
{
VideoIndex = 0;
pVideoStream = avformat_new_stream(pFormatCtx_Out, NULL);
if (!pVideoStream)
{
printf("can not new stream for output!\n");
return -1;
}
outVideoCodecCtx = avcodec_alloc_context3(outAVCodec);
if ( !outVideoCodecCtx )
{
printf("Error : avcodec_alloc_context3()\n");
return -1;
}
//set codec context param
outVideoCodecCtx = pVideoStream->codec;
outVideoCodecCtx->codec_id = AV_CODEC_ID_MPEG4;
outVideoCodecCtx->width = pFormatCtx_Video->streams[0]->codec->width;
outVideoCodecCtx->height = pFormatCtx_Video->streams[0]->codec->height;
outVideoCodecCtx->time_base = pFormatCtx_Video->streams[0]->codec->time_base;
outVideoCodecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
outVideoCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
if (codec_id == AV_CODEC_ID_H264)
{
av_opt_set(outVideoCodecCtx->priv_data, "preset", "slow", 0);
}
outAVCodec = avcodec_find_encoder(AV_CODEC_ID_MPEG4);
if( !outAVCodec )
{
printf("\n\nError : avcodec_find_encoder()");
return -1;
}
if (pFormatCtx_Out->oformat->flags & AVFMT_GLOBALHEADER)
outVideoCodecCtx->flags |=CODEC_FLAG_GLOBAL_HEADER;
if ((avcodec_open2(outVideoCodecCtx,outAVCodec, NULL)) < 0)
{
printf("can not open the encoder\n");
return -1;
}
}
if(pFormatCtx_Audio->streams[0]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
AVCodecContext *pOutputCodecCtx;
AudioIndex = 1;
pAudioStream = avformat_new_stream(pFormatCtx_Out, NULL);
pAudioStream->codec->codec = avcodec_find_encoder(pFormatCtx_Out->oformat->audio_codec);
pOutputCodecCtx = pAudioStream->codec;
pOutputCodecCtx->sample_rate = pFormatCtx_Audio->streams[0]->codec->sample_rate;
pOutputCodecCtx->channel_layout = pFormatCtx_Out->streams[0]->codec->channel_layout;
pOutputCodecCtx->channels = av_get_channel_layout_nb_channels(pAudioStream->codec->channel_layout);
if(pOutputCodecCtx->channel_layout == 0)
{
pOutputCodecCtx->channel_layout = AV_CH_LAYOUT_STEREO;
pOutputCodecCtx->channels = av_get_channel_layout_nb_channels(pOutputCodecCtx->channel_layout);
}
pOutputCodecCtx->sample_fmt = pAudioStream->codec->codec->sample_fmts[0];
AVRational time_base={1, pAudioStream->codec->sample_rate};
pAudioStream->time_base = time_base;
//audioCodecCtx->time_base = time_base;
pOutputCodecCtx->codec_tag = 0;
if (pFormatCtx_Out->oformat->flags & AVFMT_GLOBALHEADER)
pOutputCodecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER;
if (avcodec_open2(pOutputCodecCtx, pOutputCodecCtx->codec, 0) < 0)
{
printf("编码器打开失败,退出程序\n");
return -1;
}
}
if (!(pFormatCtx_Out->oformat->flags & AVFMT_NOFILE))
{
if(avio_open(&pFormatCtx_Out->pb, outFileName, AVIO_FLAG_WRITE) < 0)
{
printf("can not open output file handle!\n");
return -1;
}
}
if(avformat_write_header(pFormatCtx_Out, NULL) < 0)
{
printf("can not write the header of the output file!\n");
return -1;
}
return 0;
}
int _tmain(int argc, _TCHAR* argv[])
{
av_register_all();
avdevice_register_all();
if (OpenVideoCapture() < 0)
{
return -1;
}
if (OpenAudioCapture() < 0)
{
return -1;
}
if (OpenOutPut() < 0)
{
return -1;
}
// int fps;
/*printf("输入帧率:");
scanf_s("%d",&fps);
if ( NULL == fps)
{
fps = 10;
}*/
InitializeCriticalSection(&VideoSection);
InitializeCriticalSection(&AudioSection);
AVFrame *picture = av_frame_alloc();
int size = avpicture_get_size(pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width, pFormatCtx_Out->streams[VideoIndex]->codec->height);
picture_buf = new uint8_t[size];
avpicture_fill((AVPicture *)picture, picture_buf,
pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
//star cap screen thread
CreateThread( NULL, 0, ScreenCapThreadProc, 0, 0, NULL);
//star cap audio thread
CreateThread( NULL, 0, AudioCapThreadProc, 0, 0, NULL);
int64_t cur_pts_v=0,cur_pts_a=0;
int VideoFrameIndex = 0, AudioFrameIndex = 0;
while(1)
{
if (_kbhit() != 0 && bCap)
{
bCap = false;
Sleep(2000);
}
if (fifo_audio && fifo_video)
{
int sizeAudio = av_audio_fifo_size(fifo_audio);
int sizeVideo = av_fifo_size(fifo_video);
//缓存数据写完就结束循环
if (av_audio_fifo_size(fifo_audio) <= pFormatCtx_Out->streams[AudioIndex]->codec->frame_size &&
av_fifo_size(fifo_video) <= frame_size && !bCap)
{
break;
}
}
if(av_compare_ts(cur_pts_v, pFormatCtx_Out->streams[VideoIndex]->time_base,
cur_pts_a,pFormatCtx_Out->streams[AudioIndex]->time_base) <= 0)
{
if (av_fifo_size(fifo_video) < frame_size && !bCap)
{
cur_pts_v = 0x7fffffffffffffff;
}
if(av_fifo_size(fifo_video) >= size)
{
EnterCriticalSection(&VideoSection);
av_fifo_generic_read(fifo_video, picture_buf, size, NULL); //将数据从avfifobuffer馈送到用户提供的回调。
LeaveCriticalSection(&VideoSection);
avpicture_fill((AVPicture *)picture, picture_buf,
pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height); //根据指定的图像参数和提供的图像数据缓冲区设置图片字段。
//pts = n * ((1 / timbase)/ fps);
//picture->pts = VideoFrameIndex * ((pFormatCtx_Video->streams[0]->time_base.den / pFormatCtx_Video->streams[0]->time_base.num) / 24);
picture->pts = VideoFrameIndex * ((outVideoCodecCtx->time_base.den * 100000 / outVideoCodecCtx->time_base.num) / 180);
int got_picture = 0;
AVPacket pkt;
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
//从帧中获取输入的原始视频数据
int ret = avcodec_encode_video2(pFormatCtx_Out->streams[VideoIndex]->codec, &pkt, picture, &got_picture);
if(ret < 0)
{
continue;
}
if (got_picture==1)
{
pkt.stream_index = VideoIndex;
/*int count = 1;
pkt.pts = pkt.dts = count * ((pFormatCtx_Video->streams[0]->time_base.den / pFormatCtx_Video->streams[0]->time_base.num) / 15);
count++;*/
//x = pts * (timebase1.num / timebase1.den )* (timebase2.den / timebase2.num);
pkt.pts = av_rescale_q_rnd(pkt.pts, pFormatCtx_Video->streams[0]->time_base,
pFormatCtx_Out->streams[VideoIndex]->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, pFormatCtx_Video->streams[0]->time_base,
pFormatCtx_Out->streams[VideoIndex]->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
pkt.duration = ((pFormatCtx_Out->streams[0]->time_base.den / pFormatCtx_Out->streams[0]->time_base.num) / 60);
//pkt.duration = 1000/60;
//pkt.pts = pkt.dts = Count * (ofmt_ctx->streams[stream_index]->time_base.den) /ofmt_ctx->streams[stream_index]->time_base.num / 10;
//Count++;
cur_pts_v = pkt.pts;
ret = av_interleaved_write_frame(pFormatCtx_Out, &pkt);
//delete[] pkt.data;
av_free_packet(&pkt);
}
VideoFrameIndex++;
}
}
else
{
if (NULL == fifo_audio)
{
continue;//还未初始化fifo
}
if (av_audio_fifo_size(fifo_audio) < pFormatCtx_Out->streams[AudioIndex]->codec->frame_size && !bCap)
{
cur_pts_a = 0x7fffffffffffffff;
}
if(av_audio_fifo_size(fifo_audio) >=
(pFormatCtx_Out->streams[AudioIndex]->codec->frame_size > 0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size : 1024))
{
AVFrame *frame;
frame = av_frame_alloc();
frame->nb_samples = pFormatCtx_Out->streams[AudioIndex]->codec->frame_size>0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size: 1024;
frame->channel_layout = pFormatCtx_Out->streams[AudioIndex]->codec->channel_layout;
frame->format = pFormatCtx_Out->streams[AudioIndex]->codec->sample_fmt;
frame->sample_rate = pFormatCtx_Out->streams[AudioIndex]->codec->sample_rate;
av_frame_get_buffer(frame, 0);
EnterCriticalSection(&AudioSection);
av_audio_fifo_read(fifo_audio, (void **)frame->data,
(pFormatCtx_Out->streams[AudioIndex]->codec->frame_size > 0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size : 1024));
LeaveCriticalSection(&AudioSection);
AVPacket pkt_out;
av_init_packet(&pkt_out);
int got_picture = -1;
pkt_out.data = NULL;
pkt_out.size = 0;
frame->pts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
if (avcodec_encode_audio2(pFormatCtx_Out->streams[AudioIndex]->codec, &pkt_out, frame, &got_picture) < 0)
{
printf("can not decoder a frame");
}
av_frame_free(&frame);
if (got_picture)
{
pkt_out.stream_index = AudioIndex;
pkt_out.pts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
pkt_out.dts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
pkt_out.duration = pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
cur_pts_a = pkt_out.pts;
int ret = av_interleaved_write_frame(pFormatCtx_Out, &pkt_out);
av_free_packet(&pkt_out);
}
AudioFrameIndex++;
}
}
}
delete[] picture_buf;
av_fifo_free(fifo_video);
av_audio_fifo_free(fifo_audio);
av_write_trailer(pFormatCtx_Out);
avio_close(pFormatCtx_Out->pb);
avformat_free_context(pFormatCtx_Out);
if (pFormatCtx_Video != NULL)
{
avformat_close_input(&pFormatCtx_Video);
pFormatCtx_Video = NULL;
}
if (pFormatCtx_Audio != NULL)
{
avformat_close_input(&pFormatCtx_Audio);
pFormatCtx_Audio = NULL;
}
return 0;
}
DWORD WINAPI ScreenCapThreadProc( LPVOID lpParam )
{
AVPacket packet;
int got_picture;
AVFrame *pFrame;
pFrame=av_frame_alloc();
AVFrame *picture = av_frame_alloc();
int size = avpicture_get_size(pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
avpicture_fill((AVPicture *)picture, picture_buf,
pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
FILE *p = NULL;
p = fopen("proc_test.yuv", "wb+");
av_init_packet(&packet);
int height = pFormatCtx_Out->streams[VideoIndex]->codec->height;
int width = pFormatCtx_Out->streams[VideoIndex]->codec->width;
int y_size=height*width;
while(bCap)
{
packet.data = NULL;
packet.size = 0;
if (av_read_frame(pFormatCtx_Video, &packet) < 0)
{
continue;
}
if(packet.stream_index == 0)
{
if (avcodec_decode_video2(pCodecCtx_Video, pFrame, &got_picture, &packet) < 0)
{
printf("Decode Error.(解码错误)\n");
continue;
}
if (got_picture)
{
sws_scale(img_convert_ctx,
(const uint8_t* const*)pFrame->data,
pFrame->linesize,
0,
pFormatCtx_Out->streams[VideoIndex]->codec->height,
picture->data,
picture->linesize);
if (av_fifo_space(fifo_video) >= size)
{
EnterCriticalSection(&VideoSection);
av_fifo_generic_write(fifo_video, picture->data[0], y_size, NULL);
av_fifo_generic_write(fifo_video, picture->data[1], y_size/4, NULL);
av_fifo_generic_write(fifo_video, picture->data[2], y_size/4, NULL);
LeaveCriticalSection(&VideoSection);
}
}
}
av_free_packet(&packet);
}
av_frame_free(&pFrame);
av_frame_free(&picture);
return 0;
}
DWORD WINAPI AudioCapThreadProc( LPVOID lpParam )
{
AVPacket pkt;
AVFrame *frame;
frame = av_frame_alloc();
int gotframe;
while(bCap)
{
pkt.data = NULL;
pkt.size = 0;
if(av_read_frame(pFormatCtx_Audio,&pkt) < 0)
{
continue;
}
if (avcodec_decode_audio4(pFormatCtx_Audio->streams[0]->codec, frame, &gotframe, &pkt) < 0)
{
av_frame_free(&frame);
printf("can not decoder a frame");
break;
}
av_free_packet(&pkt);
if (!gotframe)
{
printf("没有获取到数据,继续下一次");
continue;
}
if (NULL == fifo_audio)
{
fifo_audio = av_audio_fifo_alloc(pFormatCtx_Audio->streams[0]->codec->sample_fmt,
pFormatCtx_Audio->streams[0]->codec->channels, 30 * frame->nb_samples);
}
int buf_space = av_audio_fifo_space(fifo_audio);
if (av_audio_fifo_space(fifo_audio) >= frame->nb_samples)
{
EnterCriticalSection(&AudioSection);
av_audio_fifo_write(fifo_audio, (void **)frame->data, frame->nb_samples);
LeaveCriticalSection(&AudioSection);
}
}
av_frame_free(&frame);
return 0;
}
Maybe there is another way to calculate PTS and DTS
I hope whatever the frame rate is,video playback speed is right.Not too fast or too slow.
Finally, I found out the reason for the problem.
The frame rate of video generated by video recording is determined by the recorded video stream.My computer's maximum frame rate for gdigrab is 10 frames,so if I set it more than 10 frames, the playback speed will be fast.And if i set it less than 10 frames, the playback speed will be slow.But i run code on my computer where I play games.Whatever i choose 10 frames or 15 frames,the palyback speed will be correct.
Still,i don't know the reason why my gdigrab's framerate is only 0-10 frames.There are a number of factors that affect the frame rate of video recording,such as CPU Graphics, card,display and Memory.
Here is the final code
capture screen and audio with FFMPEG
#include "stdafx.h"
#ifdef __cplusplus
extern "C"
{
#endif
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavdevice/avdevice.h"
#include "libavutil/audio_fifo.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavutil/imgutils.h"
#include "libavutil/mathematics.h"
#include "libavutil/samplefmt.h"
#include "libavutil/time.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
#include "libavutil/file.h"
#include "libavutil/mem.h"
#include "libavutil/frame.h"
#include "libavfilter/avfilter.h"
#include "libswresample/swresample.h"
#pragma comment(lib, "avcodec.lib")
#pragma comment(lib, "avformat.lib")
#pragma comment(lib, "avutil.lib")
#pragma comment(lib, "avdevice.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "postproc.lib")
#pragma comment(lib, "swresample.lib")
#pragma comment(lib, "swscale.lib")
#ifdef __cplusplus
};
#endif
AVFormatContext *pFormatCtx_Video = NULL, *pFormatCtx_Audio = NULL, *pFormatCtx_Out = NULL;
AVCodecContext *outVideoCodecCtx = NULL;
AVCodecContext *outAudioCodecCtx = NULL;
AVStream *pVideoStream = NULL, *pAudioStream = NULL;
AVCodec *outAVCodec;
AVCodec *outAudioCodec;
AVCodecContext *pCodecCtx_Video;
AVCodec *pCodec_Video;
AVFifoBuffer *fifo_video = NULL;
AVAudioFifo *fifo_audio = NULL;
int VideoIndex, AudioIndex;
int codec_id;
CRITICAL_SECTION AudioSection, VideoSection;
SwsContext *img_convert_ctx;
int frame_size = 0;
uint8_t *picture_buf = NULL;
bool bCap = true;
DWORD WINAPI ScreenCapThreadProc( LPVOID lpParam );
DWORD WINAPI AudioCapThreadProc( LPVOID lpParam );
static char *dup_wchar_to_utf8(wchar_t *w)
{
char *s = NULL;
int l = WideCharToMultiByte(CP_UTF8, 0, w, -1, 0, 0, 0, 0);
s = (char *) av_malloc(l);
if (s)
WideCharToMultiByte(CP_UTF8, 0, w, -1, s, l, 0, 0);
return s;
}
int OpenVideoCapture()
{
int fps = 10;
char opt;
printf("选择视频播放帧率:\n");
printf("A 5帧\n");
printf("B 10帧\n");
printf("C 15帧\n");
printf("D 20帧\n");
printf("E 25帧\n");
printf("F 30帧\n");
opt = getchar();
AVDictionary *options = NULL;
switch (opt)
{
case 'A':
fps = 5;
av_dict_set(&options, "framerate", "5", 0);
break;
case 'a':
fps = 5;
av_dict_set(&options, "framerate", "5", 0);
break;
case 'B':
fps = 10;
av_dict_set(&options, "framerate", "10", 0);
break;
case 'b':
fps = 10;
av_dict_set(&options, "framerate", "10", 0);
break;
case 'C':
fps = 15;
av_dict_set(&options, "framerate", "15", 0);
break;
case 'c':
fps = 15;
av_dict_set(&options, "framerate", "15", 0);
break;
case 'D':
fps = 20;
av_dict_set(&options, "framerate", "20", 0);
break;
case 'd':
fps = 20;
av_dict_set(&options, "framerate", "20", 0);
break;
case 'E':
fps = 25;
av_dict_set(&options, "framerate", "25", 0);
break;
case 'e':
fps = 25;
av_dict_set(&options, "framerate", "25", 0);
break;
case 'F':
fps = 30;
av_dict_set(&options, "framerate", "30", 0);
break;
case 'f':
fps = 30;
av_dict_set(&options, "framerate", "30", 0);
break;
default:
printf("选项输入错误\n");
return -1;
}
//AVDictionary *options = NULL;
//av_dict_set(&options, "framerate", "15", 0);
AVInputFormat *ifmt=av_find_input_format("gdigrab");
if(avformat_open_input(&pFormatCtx_Video, "desktop", ifmt, &options) != 0)
{
printf("Couldn't open input stream.(无法打开视频输入流)\n");
return -1;
}
if(avformat_find_stream_info(pFormatCtx_Video,NULL)<0)
{
printf("Couldn't find stream information.(无法获取视频流信息)\n");
return -1;
}
if (pFormatCtx_Video->streams[0]->codec->codec_type != AVMEDIA_TYPE_VIDEO)
{
printf("Couldn't find video stream information.(无法获取视频流信息)\n");
return -1;
}
pCodecCtx_Video = pFormatCtx_Video->streams[0]->codec;
pCodec_Video = avcodec_find_decoder(pCodecCtx_Video->codec_id);
if(pCodec_Video == NULL)
{
printf("Codec not found.(没有找到解码器)\n");
return -1;
}
if(avcodec_open2(pCodecCtx_Video, pCodec_Video, NULL) < 0)
{
printf("Could not open codec.(无法打开解码器)\n");
return -1;
}
av_dump_format(pFormatCtx_Video, 0, NULL, 0);
img_convert_ctx = sws_getContext(pCodecCtx_Video->width,
pCodecCtx_Video->height,
pCodecCtx_Video->pix_fmt,
pCodecCtx_Video->width,
pCodecCtx_Video->height,
PIX_FMT_YUV420P,
SWS_BICUBIC, NULL, NULL, NULL);
frame_size = avpicture_get_size(pCodecCtx_Video->pix_fmt, pCodecCtx_Video->width, pCodecCtx_Video->height);
fifo_video = av_fifo_alloc(60 * avpicture_get_size(AV_PIX_FMT_YUV420P, pCodecCtx_Video->width, pCodecCtx_Video->height));
av_dict_free(&options);
return 0;
}
int OpenAudioCapture()
{
AVInputFormat *pAudioInputFmt = av_find_input_format("dshow");
AVDictionary *opt = NULL;
char * psDevName = dup_wchar_to_utf8(L"audio=virtual-audio-capturer");
if (avformat_open_input(&pFormatCtx_Audio, psDevName, pAudioInputFmt, &opt) < 0)
{
printf("Couldn't open input stream.(无法打开音频输入流)\n");
return -1;
}
if(avformat_find_stream_info(pFormatCtx_Audio,NULL)<0)
return -1;
if(pFormatCtx_Audio->streams[0]->codec->codec_type != AVMEDIA_TYPE_AUDIO)
{
printf("Couldn't find video stream information.(无法获取音频流信息)\n");
return -1;
}
AVCodec *tmpCodec = avcodec_find_decoder(pFormatCtx_Audio->streams[0]->codec->codec_id);
if(0 > avcodec_open2(pFormatCtx_Audio->streams[0]->codec, tmpCodec, NULL))
{
printf("can not find or open audio decoder!\n");
}
av_dump_format(pFormatCtx_Audio, 0, NULL, 0);
return 0;
}
int OpenOutPut()
{
AVStream *pVideoStream = NULL, *pAudioStream = NULL;
const char *outFileName = "test.mp4";
avformat_alloc_output_context2(&pFormatCtx_Out, NULL, NULL, outFileName);
if (pFormatCtx_Video->streams[0]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
{
VideoIndex = 0;
pVideoStream = avformat_new_stream(pFormatCtx_Out, NULL);
if (!pVideoStream)
{
printf("can not new stream for output!\n");
return -1;
}
outVideoCodecCtx = avcodec_alloc_context3(outAVCodec);
if ( !outVideoCodecCtx )
{
printf("Error : avcodec_alloc_context3()\n");
return -1;
}
outVideoCodecCtx = pVideoStream->codec;
outVideoCodecCtx->codec_id = AV_CODEC_ID_MPEG4;
outVideoCodecCtx->width = pFormatCtx_Video->streams[0]->codec->width;
outVideoCodecCtx->height = pFormatCtx_Video->streams[0]->codec->height;
outVideoCodecCtx->time_base = pFormatCtx_Video->streams[0]->codec->time_base;;
outVideoCodecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
outVideoCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
if (codec_id == AV_CODEC_ID_H264)
{
av_opt_set(outVideoCodecCtx->priv_data, "preset", "slow", 0);
}
outAVCodec = avcodec_find_encoder(AV_CODEC_ID_MPEG4);
if( !outAVCodec )
{
printf("\n\nError : avcodec_find_encoder()");
return -1;
}
if (pFormatCtx_Out->oformat->flags & AVFMT_GLOBALHEADER)
outVideoCodecCtx->flags |=CODEC_FLAG_GLOBAL_HEADER;
if ((avcodec_open2(outVideoCodecCtx,outAVCodec, NULL)) < 0)
{
printf("can not open the encoder\n");
return -1;
}
}
if(pFormatCtx_Audio->streams[0]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
AVCodecContext *pOutputCodecCtx;
AudioIndex = 1;
pAudioStream = avformat_new_stream(pFormatCtx_Out, NULL);
pAudioStream->codec->codec = avcodec_find_encoder(pFormatCtx_Out->oformat->audio_codec);
pOutputCodecCtx = pAudioStream->codec;
pOutputCodecCtx->sample_rate = pFormatCtx_Audio->streams[0]->codec->sample_rate;
pOutputCodecCtx->channel_layout = pFormatCtx_Out->streams[0]->codec->channel_layout;
pOutputCodecCtx->channels = av_get_channel_layout_nb_channels(pAudioStream->codec->channel_layout);
if(pOutputCodecCtx->channel_layout == 0)
{
pOutputCodecCtx->channel_layout = AV_CH_LAYOUT_STEREO;
pOutputCodecCtx->channels = av_get_channel_layout_nb_channels(pOutputCodecCtx->channel_layout);
}
pOutputCodecCtx->sample_fmt = pAudioStream->codec->codec->sample_fmts[0];
AVRational time_base={1, pAudioStream->codec->sample_rate};
pAudioStream->time_base = time_base;
pOutputCodecCtx->codec_tag = 0;
if (pFormatCtx_Out->oformat->flags & AVFMT_GLOBALHEADER)
pOutputCodecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER;
if (avcodec_open2(pOutputCodecCtx, pOutputCodecCtx->codec, 0) < 0)
{
printf("编码器打开失败,退出程序\n");
return -1;
}
}
if (!(pFormatCtx_Out->oformat->flags & AVFMT_NOFILE))
{
if(avio_open(&pFormatCtx_Out->pb, outFileName, AVIO_FLAG_WRITE) < 0)
{
printf("can not open output file handle!\n");
return -1;
}
}
if(avformat_write_header(pFormatCtx_Out, NULL) < 0)
{
printf("can not write the header of the output file!\n");
return -1;
}
return 0;
}
int _tmain(int argc, _TCHAR* argv[])
{
av_register_all();
avdevice_register_all();
if (OpenVideoCapture() < 0)
{
return -1;
}
if (OpenAudioCapture() < 0)
{
return -1;
}
if (OpenOutPut() < 0)
{
return -1;
}
InitializeCriticalSection(&VideoSection);
InitializeCriticalSection(&AudioSection);
AVFrame *picture = av_frame_alloc();
int size = avpicture_get_size(pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width, pFormatCtx_Out->streams[VideoIndex]->codec->height);
picture_buf = new uint8_t[size];
avpicture_fill((AVPicture *)picture, picture_buf,
pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
//star cap screen thread
CreateThread( NULL, 0, ScreenCapThreadProc, 0, 0, NULL);
//star cap audio thread
CreateThread( NULL, 0, AudioCapThreadProc, 0, 0, NULL);
int64_t cur_pts_v=0,cur_pts_a=0;
int64_t VideoFrameIndex = 0, AudioFrameIndex = 0;
int64_t count = 1;
int64_t video_pre_pts = 0;
while(1)
{
if (_kbhit() != 0 && bCap)
{
bCap = false;
Sleep(2000);
}
if (fifo_audio && fifo_video)
{
int sizeAudio = av_audio_fifo_size(fifo_audio);
int sizeVideo = av_fifo_size(fifo_video);
//缓存数据写完就结束循环
if (av_audio_fifo_size(fifo_audio) <= pFormatCtx_Out->streams[AudioIndex]->codec->frame_size &&
av_fifo_size(fifo_video) <= frame_size && !bCap)
{
break;
}
}
if(av_compare_ts(cur_pts_v, pFormatCtx_Out->streams[VideoIndex]->time_base, cur_pts_a,pFormatCtx_Out->streams[AudioIndex]->time_base) <= 0)
{
if (av_fifo_size(fifo_video) < frame_size && !bCap)
{
cur_pts_v = 0x7fffffffffffffff;
}
if(av_fifo_size(fifo_video) >= size)
{
//将数据从avfifobuffer馈送到用户提供的回调。
EnterCriticalSection(&VideoSection);
av_fifo_generic_read(fifo_video, picture_buf, size, NULL);
LeaveCriticalSection(&VideoSection);
//根据指定的图像参数和提供的图像数据缓冲区设置图片字段。
avpicture_fill((AVPicture *)picture, picture_buf,
pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
//pts = n * ((1 / timbase)/ fps);
//picture->pts = VideoFrameIndex * ((pFormatCtx_Video->streams[0]->time_base.den / pFormatCtx_Video->streams[0]->time_base.num) / 15);
picture->pts = av_rescale_q(VideoFrameIndex,outVideoCodecCtx->time_base,pFormatCtx_Video->streams[0]->time_base);
printf("picture->pts: %d\n",picture->pts);
int got_picture = 0;
AVPacket pkt;
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
//从帧中获取输入的原始视频数据
int ret = avcodec_encode_video2(pFormatCtx_Out->streams[VideoIndex]->codec, &pkt, picture, &got_picture);
if(ret < 0)
{
continue;
}
if (got_picture==1)
{
pkt.stream_index = VideoIndex;
//pFormatCtx_Video //pFormatCtx_Out
pkt.pts = av_rescale_q_rnd(pkt.pts, pFormatCtx_Video->streams[0]->time_base, pFormatCtx_Out->streams[VideoIndex]->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
printf("pkt.pts = %d\n",pkt.pts);
pkt.dts = av_rescale_q_rnd(pkt.dts, pFormatCtx_Video->streams[0]->time_base, pFormatCtx_Out->streams[VideoIndex]->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
printf("pkt.dts = %d\n",pkt.dts);
pkt.duration = ((pFormatCtx_Out->streams[0]->time_base.den / pFormatCtx_Out->streams[0]->time_base.num) / outVideoCodecCtx->time_base.den);
//pkt.duration = 1;
//pkt.duration = av_rescale_q(pkt.duration,outVideoCodecCtx->time_base,pFormatCtx_Video->streams[0]->time_base);
printf("pkt.duration = %d\n",pkt.duration);
pkt.pos = -1;
cur_pts_v = pkt.pts;
ret = av_interleaved_write_frame(pFormatCtx_Out, &pkt);
if (ret<0)
{
printf("Error muxing packet\n");
break;
}
av_free_packet(&pkt);
}
VideoFrameIndex++;
}
}
else
{
if (NULL == fifo_audio)
{
continue;//还未初始化fifo
}
if (av_audio_fifo_size(fifo_audio) < pFormatCtx_Out->streams[AudioIndex]->codec->frame_size && !bCap)
{
cur_pts_a = 0x7fffffffffffffff;
}
if(av_audio_fifo_size(fifo_audio) >=
(pFormatCtx_Out->streams[AudioIndex]->codec->frame_size > 0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size : 1024))
{
AVFrame *frame;
frame = av_frame_alloc();
frame->nb_samples = pFormatCtx_Out->streams[AudioIndex]->codec->frame_size>0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size: 1024;
frame->channel_layout = pFormatCtx_Out->streams[AudioIndex]->codec->channel_layout;
frame->format = pFormatCtx_Out->streams[AudioIndex]->codec->sample_fmt;
frame->sample_rate = pFormatCtx_Out->streams[AudioIndex]->codec->sample_rate;
av_frame_get_buffer(frame, 0);
EnterCriticalSection(&AudioSection);
av_audio_fifo_read(fifo_audio, (void **)frame->data,
(pFormatCtx_Out->streams[AudioIndex]->codec->frame_size > 0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size : 1024));
LeaveCriticalSection(&AudioSection);
AVPacket pkt_out;
av_init_packet(&pkt_out);
int got_picture = -1;
pkt_out.data = NULL;
pkt_out.size = 0;
frame->pts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
if (avcodec_encode_audio2(pFormatCtx_Out->streams[AudioIndex]->codec, &pkt_out, frame, &got_picture) < 0)
{
printf("can not decoder a frame");
}
av_frame_free(&frame);
if (got_picture)
{
pkt_out.stream_index = AudioIndex;
pkt_out.pts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
pkt_out.dts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
pkt_out.duration = pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
cur_pts_a = pkt_out.pts;
int ret = av_interleaved_write_frame(pFormatCtx_Out, &pkt_out);
if (ret<0)
{
printf("Error muxing packet\n");
break;
}
av_free_packet(&pkt_out);
}
AudioFrameIndex++;
}
}
}
delete[] picture_buf;
av_fifo_free(fifo_video);
av_audio_fifo_free(fifo_audio);
av_write_trailer(pFormatCtx_Out);
avio_close(pFormatCtx_Out->pb);
avformat_free_context(pFormatCtx_Out);
if (pFormatCtx_Video != NULL)
{
avformat_close_input(&pFormatCtx_Video);
pFormatCtx_Video = NULL;
}
if (pFormatCtx_Audio != NULL)
{
avformat_close_input(&pFormatCtx_Audio);
pFormatCtx_Audio = NULL;
}
return 0;
}
DWORD WINAPI ScreenCapThreadProc( LPVOID lpParam )
{
AVPacket packet;
int got_picture;
AVFrame *pFrame;
pFrame=av_frame_alloc();
AVFrame *picture = av_frame_alloc();
int size = avpicture_get_size(pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
avpicture_fill((AVPicture *)picture, picture_buf,
pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt,
pFormatCtx_Out->streams[VideoIndex]->codec->width,
pFormatCtx_Out->streams[VideoIndex]->codec->height);
av_init_packet(&packet);
int height = pFormatCtx_Out->streams[VideoIndex]->codec->height;
int width = pFormatCtx_Out->streams[VideoIndex]->codec->width;
int y_size=height*width;
while(bCap)
{
packet.data = NULL;
packet.size = 0;
if (av_read_frame(pFormatCtx_Video, &packet) < 0)
{
continue;
}
if(packet.stream_index == 0)
{
if (avcodec_decode_video2(pCodecCtx_Video, pFrame, &got_picture, &packet) < 0)
{
printf("Decode Error.(解码错误)\n");
continue;
}
if (got_picture)
{
sws_scale(img_convert_ctx,
(const uint8_t* const*)pFrame->data,
pFrame->linesize,
0,
pFormatCtx_Out->streams[VideoIndex]->codec->height,
picture->data,
picture->linesize);
if (av_fifo_space(fifo_video) >= size)
{
EnterCriticalSection(&VideoSection);
av_fifo_generic_write(fifo_video, picture->data[0], y_size, NULL);
av_fifo_generic_write(fifo_video, picture->data[1], y_size/4, NULL);
av_fifo_generic_write(fifo_video, picture->data[2], y_size/4, NULL);
LeaveCriticalSection(&VideoSection);
}
}
}
av_free_packet(&packet);
}
av_frame_free(&pFrame);
av_frame_free(&picture);
return 0;
}
DWORD WINAPI AudioCapThreadProc( LPVOID lpParam )
{
AVPacket pkt;
AVFrame *frame;
frame = av_frame_alloc();
int gotframe;
while(bCap)
{
pkt.data = NULL;
pkt.size = 0;
if(av_read_frame(pFormatCtx_Audio,&pkt) < 0)
{
continue;
}
if (avcodec_decode_audio4(pFormatCtx_Audio->streams[0]->codec, frame, &gotframe, &pkt) < 0)
{
av_frame_free(&frame);
printf("can not decoder a frame");
break;
}
av_free_packet(&pkt);
if (!gotframe)
{
printf("没有获取到数据,继续下一次");
continue;
}
if (NULL == fifo_audio)
{
fifo_audio = av_audio_fifo_alloc(pFormatCtx_Audio->streams[0]->codec->sample_fmt,
pFormatCtx_Audio->streams[0]->codec->channels, 30 * frame->nb_samples);
}
int buf_space = av_audio_fifo_space(fifo_audio);
if (av_audio_fifo_space(fifo_audio) >= frame->nb_samples)
{
EnterCriticalSection(&AudioSection);
av_audio_fifo_write(fifo_audio, (void **)frame->data, frame->nb_samples);
LeaveCriticalSection(&AudioSection);
}
}
av_frame_free(&frame);
return 0;
}

FFMPEG AAC encoder issue

I am trying to capture and encode audio data, I am encoding audio using
FFMPEG AAC and to capture PCM data I used ALSA, Capturing part is working in my case, However, AAC encoder is not working.
I am trying to play test.aac file using
ffplay test.aac
but it contains lots of noise.
Attaching code for aac encoder :
#include "AudioEncoder.h"
void* AudioEncoder::run(void *ctx)
{
return ((AudioEncoder *)ctx)->execute();
}
static int frameCount = 0;
void* AudioEncoder::execute(void)
{
float buf[size], *temp;
int totalSize = 0;
int fd = open("in.pcm", O_CREAT| O_RDWR, 0666);
int frameSize = 128 * snd_pcm_format_width(SND_PCM_FORMAT_FLOAT) / 8 * 2;
av_new_packet(&pkt,size);
cout << size << endl;
while (!Main::stopThread)
{
temp = (Main::fbAudio)->dequeue();
memcpy(buf + totalSize, temp, frameSize);
write(fd, temp, frameSize); // Can play in.pcm with no noise in it.
totalSize += frameSize;
delete temp;
if (totalSize >= size)
{
totalSize = 0;
//frame_buf = (uint8_t *) buf;
pFrame->data[0] = (uint8_t *)buf; //PCM Data
pFrame->pts=frameCount;
frameCount++;
got_frame=0;
//Encode
ret = avcodec_encode_audio2(pCodecCtx, &pkt,pFrame, &got_frame);
if(ret < 0){
cerr << "Failed to encode!\n";
return NULL;
}
if (got_frame==1){
printf("Succeed to encode 1 frame! \tsize:%5d\n",pkt.size);
pkt.stream_index = audio_st->index;
#ifdef DUMP_TEST
ret = av_write_frame(pFormatCtx, &pkt);
#endif
av_free_packet(&pkt);
}
//memset(buf, 0, sizeof(float)*size);
}
//delete temp;
//if (buf.size() >= m_audio_output_decoder_ctx->frame_size)
/* encode the audio*/
}
close(fd);
Main::stopThread = true;
return NULL;
}
int AudioEncoder::flush_encoder(AVFormatContext *fmt_ctx,unsigned int stream_index){
int ret;
int got_frame;
AVPacket enc_pkt;
if (!(fmt_ctx->streams[stream_index]->codec->codec->capabilities &
CODEC_CAP_DELAY))
return 0;
while (1) {
enc_pkt.data = NULL;
enc_pkt.size = 0;
av_init_packet(&enc_pkt);
ret = avcodec_encode_audio2 (fmt_ctx->streams[stream_index]->codec, &enc_pkt,
NULL, &got_frame);
av_frame_free(NULL);
if (ret < 0)
break;
if (!got_frame){
ret=0;
break;
}
printf("Flush Encoder: Succeed to encode 1 frame!\tsize:%5d\n",enc_pkt.size);
/* mux encoded frame */
#ifdef DUMP_TEST
ret = av_write_frame(fmt_ctx, &enc_pkt);
if (ret < 0)
break;
#endif
}
return ret;
}
void AudioEncoder::start(void)
{
pthread_t encoder;
pthread_create(&encoder, NULL, &AudioEncoder::run, this);
}
AudioEncoder::AudioEncoder() : out_file("test.aac")
{
got_frame = 0;
ret = 0;
size = 0;
av_register_all();
avcodec_register_all();
//Method 1.
pFormatCtx = avformat_alloc_context();
fmt = av_guess_format(NULL, out_file, NULL);
pFormatCtx->oformat = fmt;
#ifdef DUMP_TEST
if (avio_open(&pFormatCtx->pb,out_file, AVIO_FLAG_READ_WRITE) < 0){
cerr << "Failed to open output file!\n";
return;
}
#endif
audio_st = avformat_new_stream(pFormatCtx, 0);
if (audio_st==NULL){
return;
}
pCodecCtx = audio_st->codec;
pCodecCtx->codec_id = fmt->audio_codec;
pCodecCtx->codec_type = AVMEDIA_TYPE_AUDIO;
pCodecCtx->sample_fmt = AV_SAMPLE_FMT_FLTP;
pCodecCtx->sample_rate= 8000;
pCodecCtx->channel_layout=AV_CH_LAYOUT_STEREO;
pCodecCtx->channels = av_get_channel_layout_nb_channels(pCodecCtx->channel_layout);
// pCodecCtx->bit_rate = 64000;
#ifdef DUMP_TEST
//Show some information
av_dump_format(pFormatCtx, 0, out_file, 1);
#endif
pCodec = avcodec_find_encoder(pCodecCtx->codec_id);
if (!pCodec){
printf("Can not find encoder!\n");
return;
}
if (avcodec_open2(pCodecCtx, pCodec,NULL) < 0){
printf("Failed to open encoder!\n");
return;
}
pFrame = av_frame_alloc();
pFrame->nb_samples= pCodecCtx->frame_size;
pFrame->format= pCodecCtx->sample_fmt;
size = av_samples_get_buffer_size(NULL, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
frame_buf = (uint8_t *)av_malloc(size);
avcodec_fill_audio_frame(pFrame, pCodecCtx->channels, pCodecCtx->sample_fmt,(const uint8_t*)frame_buf, size, 1);
//Write Header
#ifdef DUMP_TEST
avformat_write_header(pFormatCtx,NULL);
#endif
}
AudioEncoder::~AudioEncoder()
{
//Flush Encoder
ret = flush_encoder(pFormatCtx,0);
if (ret < 0) {
cerr << "Flushing encoder failed\n";
return;
}
#ifdef DUMP_TEST
//Write Trailer
av_write_trailer(pFormatCtx);
#endif
//Clean
if (audio_st){
avcodec_close(audio_st->codec);
av_free(pFrame);
av_free(frame_buf);
}
avio_close(pFormatCtx->pb);
avformat_free_context(pFormatCtx);
}
Here, please ignore DUMP_TEST flag, I already enabled it.
Can some one tell me what is issue ?
Thanks,
Harshil
I am able to resolve this issue, by correctly passing buffer from ALSA to AAC encoder.
Here AAC expects buffer size of 4096 bytes, but from deque I am passing 1024 bytes which causes issue, also I updated audio channels to MONO, in place of STEREO. Attaching my working code snippet for more information:
#include "AudioEncoder.h"
void* AudioEncoder::run(void *ctx)
{
return ((AudioEncoder *)ctx)->execute();
}
static int frameCount = 0;
void* AudioEncoder::execute(void)
{
float *temp;
#ifdef DUMP_TEST
int fd = open("in.pcm", O_CREAT| O_RDWR, 0666);
#endif
int frameSize = 1024 * snd_pcm_format_width(SND_PCM_FORMAT_FLOAT) / 8 * 1;
av_new_packet(&pkt,size);
while (!Main::stopThread)
{
temp = (Main::fbAudio)->dequeue();
frame_buf = (uint8_t *) temp;
pFrame->data[0] = frame_buf;
pFrame->pts=frameCount*100;
frameCount++;
got_frame=0;
//Encode
ret = avcodec_encode_audio2(pCodecCtx, &pkt,pFrame, &got_frame);
if(ret < 0){
cerr << "Failed to encode!\n";
return NULL;
}
if (got_frame==1){
cout << "Encoded frame\n";
pkt.stream_index = audio_st->index;
#ifdef DUMP_TEST
write(fd, temp, frameSize);
ret = av_interleaved_write_frame(pFormatCtx, &pkt);
#endif
av_free_packet(&pkt);
}
delete temp;
}
#ifdef DUMP_TEST
close(fd);
#endif
Main::stopThread = true;
return NULL;
}
int AudioEncoder::flush_encoder(AVFormatContext *fmt_ctx,unsigned int stream_index){
int ret;
int got_frame;
AVPacket enc_pkt;
if (!(fmt_ctx->streams[stream_index]->codec->codec->capabilities &
CODEC_CAP_DELAY))
return 0;
while (1) {
enc_pkt.data = NULL;
enc_pkt.size = 0;
av_init_packet(&enc_pkt);
ret = avcodec_encode_audio2 (fmt_ctx->streams[stream_index]->codec, &enc_pkt,
NULL, &got_frame);
av_frame_free(NULL);
if (ret < 0)
break;
if (!got_frame){
ret=0;
break;
}
printf("Flush Encoder: Succeed to encode 1 frame!\tsize:%5d\n",enc_pkt.size);
/* mux encoded frame */
#ifdef DUMP_TEST
ret = av_write_frame(fmt_ctx, &enc_pkt);
if (ret < 0)
break;
#endif
}
return ret;
}
void AudioEncoder::start(void)
{
pthread_t encoder;
pthread_create(&encoder, NULL, &AudioEncoder::run, this);
}
AudioEncoder::AudioEncoder() : out_file("test.aac")
{
got_frame = 0;
ret = 0;
size = 0;
av_register_all();
avcodec_register_all();
//Method 1.
pFormatCtx = avformat_alloc_context();
fmt = av_guess_format(NULL, out_file, NULL);
pFormatCtx->oformat = fmt;
#ifdef DUMP_TEST
if (avio_open(&pFormatCtx->pb,out_file, AVIO_FLAG_READ_WRITE) < 0){
cerr << "Failed to open output file!\n";
return;
}
#endif
audio_st = avformat_new_stream(pFormatCtx, 0);
if (audio_st==NULL){
return;
}
pCodecCtx = audio_st->codec;
pCodecCtx->codec_id = fmt->audio_codec;
pCodecCtx->codec_type = AVMEDIA_TYPE_AUDIO;
pCodecCtx->sample_fmt = AV_SAMPLE_FMT_FLTP;
pCodecCtx->sample_rate= 8000;
pCodecCtx->channel_layout=AV_CH_LAYOUT_MONO;
pCodecCtx->channels = av_get_channel_layout_nb_channels(pCodecCtx->channel_layout);
pCodecCtx->bit_rate = 64000;
#ifdef DUMP_TEST
//Show some information
av_dump_format(pFormatCtx, 0, out_file, 1);
#endif
pCodec = avcodec_find_encoder(pCodecCtx->codec_id);
if (!pCodec){
printf("Can not find encoder!\n");
return;
}
if (avcodec_open2(pCodecCtx, pCodec,NULL) < 0){
printf("Failed to open encoder!\n");
return;
}
pFrame = av_frame_alloc();
pFrame->nb_samples= pCodecCtx->frame_size;
pFrame->format= pCodecCtx->sample_fmt;
size = av_samples_get_buffer_size(NULL, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
frame_buf = (uint8_t *)av_malloc(size);
avcodec_fill_audio_frame(pFrame, pCodecCtx->channels, pCodecCtx->sample_fmt,(const uint8_t*)frame_buf, size, 1);
//Write Header
#ifdef DUMP_TEST
avformat_write_header(pFormatCtx,NULL);
#endif
}
AudioEncoder::~AudioEncoder()
{
//Flush Encoder
ret = flush_encoder(pFormatCtx,0);
if (ret < 0) {
cerr << "Flushing encoder failed\n";
return;
}
#ifdef DUMP_TEST
//Write Trailer
av_write_trailer(pFormatCtx);
#endif
//Clean
if (audio_st){
avcodec_close(audio_st->codec);
av_free(pFrame);
av_free(frame_buf);
}
avio_close(pFormatCtx->pb);
avformat_free_context(pFormatCtx);
}

Why is the Write system call in not writing anything to the sockets file?

I am writing a simple Server Client program that exchanges the data. After I write to the socket file, the write doesn't fail or it is not even partial write. but when I check the details of the socket file using ls -l , I still see it's size as zero and server doesn't recieve anything. Can anyone help me out with what I am doing wrong in here??
This is server.c
int main()
{
int socket_fd = 0;
struct sockaddr_un addr;
int result = -1;
char buffer[MAX_BUFFER_SIZE];
printf("Creating a socket\n");
socket_fd = socket(AF_UNIX,SOCK_STREAM,0);
if(socket_fd == -1)
{
perror("SOCKET");
return 0;
}
printf("Socket has been created %d\n",socket_fd);
addr.sun_family = AF_UNIX;
strncpy(addr.sun_path,_SOCKET_PATH,sizeof(addr.sun_path)-1);
printf("PATH : %s\n",addr.sun_path);
if(remove(_SOCKET_PATH) == -1)
{
perror("REMOVE");
return 0;
}
printf("Binding the socket\n");
result = bind(socket_fd,(struct sockaddr *)&addr,sizeof(struct sockaddr_un));
if(result == -1)
{
perror("BIND");
return 0;
}
printf("Binding the socket is done\n");
printf("Listening to the socket\n");
if(listen(socket_fd,1) == -1)
{
perror("Listen");
return 0;
}
if((result = accept(socket_fd,NULL,NULL)) == -1)
{
perror("ACCEPT");
return 0;
}
printf("Connection Accepted\n");
while (1)
{
while (result = read(socket_fd,buffer,sizeof(buffer)-1) > 0)
{
printf("Client said : %s\n",buffer);
}
}
}
This is client.c
int main()
{
int socket_fd = 0;
struct sockaddr_un addr;
int result = -1;
char buffer[MAX_BUFFER_SIZE];
printf("Creating a socket\n");
socket_fd = socket(AF_UNIX,SOCK_STREAM,0);
if(socket_fd == -1)
{
perror("SOCKET");
return 0;
}
printf("Socket has been found %d\n",socket_fd);
addr.sun_family = AF_UNIX;
strncpy(addr.sun_path,_SOCKET_PATH,sizeof(addr.sun_path)-1);
printf("Connecting to the socket\n");
result = connect(socket_fd,(struct sockaddr *)&addr,sizeof(struct sockaddr_un));
if(result == -1)
{
perror("CONNECT");
return 0;
}
printf("The client is connected to the server.\n");
while (1)
{
memset(buffer,0,sizeof(buffer));
scanf("%s",buffer);
printf("DATA WRITTEN %s,%d\n",buffer,strlen(buffer)+1);
result = write(socket_fd,buffer,strlen(buffer)+1);
printf("result = %d\n",result);
sleep(5);
}
}
Thanks for any help!
if((result = accept(socket_fd,NULL,NULL)) == -1)
...
while (result = read(socket_fd,buffer,sizeof(buffer)-1) > 0)
You are trying to read from the server socket (socket_fd). Instead you need to read from the new socket returned by accept, i.e. what you call result in result = accept.... To cite from man accept:
On success, these system calls return a nonnegative integer that is a
file descriptor for the accepted socket. On error, -1 is returned,
and errno is set appropriately.

Which thread is used in the same function of multithread

In mfc code,
Declare as follow
CWinThread* _pThread[5];
DWORD _ThreadArg[5] = { HIGHEST_THREAD, // 0x00
ABOVE_AVE_THREAD, // 0x3F
NORMAL_THREAD, // 0x7F
BELOW_AVE_THREAD, // 0xBF
LOWEST_THREAD // 0xFF
};
int i;
for (i= 0; i< 5; i++)
{
_pThread[i] = AfxBeginThread(ThreadFunc,
&_ThreadArg[i],
THREAD_PRIORITY_NORMAL,
0,
//CREATE_SUSPENDED,
NULL,
NULL);
//_pThread[i]->ResumeThread();
}
// setup relative priority of threads
_pThread[0]->SetThreadPriority(THREAD_PRIORITY_HIGHEST);
_pThread[1]->SetThreadPriority(THREAD_PRIORITY_ABOVE_NORMAL);
_pThread[2]->SetThreadPriority(THREAD_PRIORITY_NORMAL);
_pThread[3]->SetThreadPriority(THREAD_PRIORITY_BELOW_NORMAL);
_pThread[4]->SetThreadPriority(THREAD_PRIORITY_LOWEST);
use same thread function
UINT CThreadPoolDlg::ThreadFuncTh1(LPVOID ThreadArg)
{
DWORD dwArg = *(DWORD*)ThreadArg;
//txtTh[0].SetWindowTextW(_T("23"));
AfxGetMainWnd()->GetDlgItem(IDC_THD1)->SetWindowText(_T("1"));
return 0;
}
How do I check which thread is in use in same function?
I get a answer as follow
_pThread[0] = AfxBeginThread(ThreadFunc,this,
THREAD_PRIORITY_NORMAL,
0,
NULL,
NULL);
_pThread[1] = AfxBeginThread(ThreadFunc,this,
THREAD_PRIORITY_NORMAL,
0,
CREATE_SUSPENDED,
NULL);
_pThread[2] = AfxBeginThread(ThreadFunc,this,
THREAD_PRIORITY_NORMAL,
0,
CREATE_SUSPENDED,
NULL);
the thread function is declare as follow
UINT CThreadPoolDlg::ThreadFunc(LPVOID ThreadArg)
{
CThreadPoolDlg* dwArg = (CThreadPoolDlg*)ThreadArg;
DWORD dThread = GetCurrentThreadId();
//txtTh[0].SetWindowTextW(_T("23"));
while(1)
{
CString strTemp;
if(dThread == dwArg->_pThread[0]->m_nThreadID)
{
AfxGetMainWnd()->GetDlgItem(IDC_THD1)->GetWindowText(strTemp);
int n = _wtoi(strTemp);
strTemp.Format(_T("%d"), ++n);
AfxGetMainWnd()->GetDlgItem(IDC_THD1)->SetWindowText(strTemp);
if(n > 5)
{
::SendMessage(dwArg->GetSafeHwnd(), OWM_MYMETHOD, 0, 0);
//dwArg->_pThread[1]->ResumeThread();
}
}
if(dThread == dwArg->_pThread[1]->m_nThreadID)
{
AfxGetMainWnd()->GetDlgItem(IDC_THD2)->GetWindowText(strTemp);
int n = _wtoi(strTemp);
strTemp.Format(_T("%d"), ++n);
AfxGetMainWnd()->GetDlgItem(IDC_THD2)->SetWindowText(strTemp);
if(n > 5)
{
dwArg->_pThread[2]->ResumeThread();
}
}
if(dThread == dwArg->_pThread[2]->m_nThreadID)
{
AfxGetMainWnd()->GetDlgItem(IDC_THD3)->GetWindowText(strTemp);
int n = _wtoi(strTemp);
strTemp.Format(_T("%d"), ++n);
AfxGetMainWnd()->GetDlgItem(IDC_THD3)->SetWindowText(strTemp);
}
Sleep(500);
}
return 0;
}
Just look at the
DWORD dThread = GetCurrentThreadId();
dThread == dwArg->_pThread[1]->m_nThreadID
then I will get current thread id...

Resources