diff --git a/.gitmodules b/.gitmodules index 870562b1..1c9502a7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "third_party/imgui"] path = third_party/imgui url = https://github.com/ocornut/imgui.git +[submodule "third_party/NVIDIAImageScaling"] + path = third_party/NVIDIAImageScaling + url = https://github.com/NVIDIAGameWorks/NVIDIAImageScaling.git diff --git a/Assets/Shader/build_nis.bat b/Assets/Shader/build_nis.bat new file mode 100644 index 00000000..c2eee052 --- /dev/null +++ b/Assets/Shader/build_nis.bat @@ -0,0 +1,17 @@ +@echo off +setlocal +pushd "%~dp0" || exit /b 1 +set "HLSL=..\..\third_party\NVIDIAImageScaling\NIS\NIS_Main.hlsl" +set "EXIT_CODE=0" +fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=0 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=32 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=1 /Fo "nis_sdr_fp16.cso" "%HLSL%" || goto :fail +fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=2 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=32 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=1 /Fo "nis_hdr_fp16.cso" "%HLSL%" || goto :fail +fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=0 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=24 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=0 /Fo "nis_sdr_fp32.cso" "%HLSL%" || goto :fail +fxc /T cs_5_0 /E main /O3 /D NIS_SCALER=1 /D NIS_HDR_MODE=2 /D NIS_BLOCK_WIDTH=32 /D NIS_BLOCK_HEIGHT=24 /D NIS_THREAD_GROUP_SIZE=128 /D NIS_USE_HALF_PRECISION=0 /Fo "nis_hdr_fp32.cso" "%HLSL%" || goto :fail +goto :cleanup + +:fail +set "EXIT_CODE=%ERRORLEVEL%" + +:cleanup +popd +endlocal & exit /b %EXIT_CODE% diff --git a/Assets/Shader/nis_hdr_fp16.cso b/Assets/Shader/nis_hdr_fp16.cso new file mode 100644 index 00000000..d2c8d84c Binary files /dev/null and b/Assets/Shader/nis_hdr_fp16.cso differ diff --git a/Assets/Shader/nis_hdr_fp32.cso b/Assets/Shader/nis_hdr_fp32.cso new file mode 100644 index 00000000..fcc664dd Binary files /dev/null and b/Assets/Shader/nis_hdr_fp32.cso differ diff --git a/Assets/Shader/nis_sdr_fp16.cso b/Assets/Shader/nis_sdr_fp16.cso new file mode 100644 index 00000000..b2bdbfab Binary files /dev/null and b/Assets/Shader/nis_sdr_fp16.cso differ diff --git a/Assets/Shader/nis_sdr_fp32.cso b/Assets/Shader/nis_sdr_fp32.cso new file mode 100644 index 00000000..1980e040 Binary files /dev/null and b/Assets/Shader/nis_sdr_fp32.cso differ diff --git a/Pages/AppPage.xaml.cpp b/Pages/AppPage.xaml.cpp index 2558f454..5fb63177 100644 --- a/Pages/AppPage.xaml.cpp +++ b/Pages/AppPage.xaml.cpp @@ -166,6 +166,7 @@ void AppPage::Connect(int appId) { config->audioConfig = host->AudioConfig; config->videoCodec = host->VideoCodec; config->playAudioOnPC = host->PlayAudioOnPC; + config->videoSuperResolution = host->VideoSuperResolution; config->enableHDR = host->EnableHDR; config->enableSOPS = host->EnableSOPS; config->framePacing = host->FramePacing; diff --git a/Pages/HostSettingsPage.xaml b/Pages/HostSettingsPage.xaml index ae0f31fa..d1a8fa19 100644 --- a/Pages/HostSettingsPage.xaml +++ b/Pages/HostSettingsPage.xaml @@ -40,6 +40,7 @@ + Resolution @@ -57,58 +58,60 @@ Bitrate (Kbps) - Audio Configuration - - Play Audio on Host PC - - Autostart - + Video Super Resolution + + Audio Configuration + + Play Audio on Host PC + + Autostart + - Video Codecs - + Video Codecs + - Enable HDR: + Enable HDR: + Name="HDR4KNote" Grid.Row="8" Grid.Column="1" Grid.ColumnSpan="2" Visibility="Collapsed"> HDR requires Xbox system resolution set to 4K. - Optimize host resolution: - + Optimize host resolution: + - Frame Pacing: - + Frame Pacing: + + Name="FramePacingImmediateDesc" Grid.Row="10" Grid.Column="2" Visibility="Collapsed"> Best for Xbox Series. Lowest latency. Renders frames at the incoming frame rate. + Name="FramePacingDisplayLockedDesc" Grid.Row="10" Grid.Column="2" Visibility="Collapsed"> Best for Xbox One. Locks rendering frame rate to refresh rate and evenly spaces frames. - Show performance stats: + Show performance stats: - Show performance graphs: + Show performance graphs: + Name="XboxOneGraphsNote" Grid.Row="12" Grid.Column="1" Grid.ColumnSpan="2" Visibility="Collapsed"> Graphs are unavailable on Xbox One when system resolution is set to 4K. - Other: - + Other: + diff --git a/State/ApplicationState.cpp b/State/ApplicationState.cpp index 611792ca..721e862d 100644 --- a/State/ApplicationState.cpp +++ b/State/ApplicationState.cpp @@ -44,6 +44,7 @@ Concurrency::task moonlight_xbox_dx::ApplicationState::Init() if (a.contains("autoStartID"))h->AutostartID = a["autoStartID"]; if (a.contains("computername")) h->ComputerName = Utils::StringFromStdString(a["computername"].get()); if (a.contains("playaudioonpc")) h->PlayAudioOnPC = a["playaudioonpc"].get(); + if (a.contains("video_super_resolution")) h->VideoSuperResolution = a["video_super_resolution"].get(); if (a.contains("enable_hdr")) h->EnableHDR = a["enable_hdr"].get(); if (a.contains("enable_sops")) h->EnableSOPS = a["enable_sops"].get(); if (a.contains("enable_stats")) h->EnableStats = a["enable_stats"].get(); @@ -104,6 +105,7 @@ Concurrency::task moonlight_xbox_dx::ApplicationState::UpdateFile() hostJson["framePacing"] = Utils::PlatformStringToStdString(host->FramePacing); hostJson["autoStartID"] = host->AutostartID; hostJson["playaudioonpc"] = host->PlayAudioOnPC; + hostJson["video_super_resolution"] = host->VideoSuperResolution; hostJson["enable_hdr"] = host->EnableHDR; hostJson["enable_sops"] = host->EnableSOPS; hostJson["enable_stats"] = host->EnableStats; diff --git a/State/MoonlightHost.h b/State/MoonlightHost.h index 52c2e126..d5633ce4 100644 --- a/State/MoonlightHost.h +++ b/State/MoonlightHost.h @@ -18,6 +18,7 @@ namespace moonlight_xbox_dx { bool loading = true; bool wolPolling = false; bool playAudioOnPC = false; + bool videoSuperResolution = false; MoonlightClient* client; int currentlyRunningAppId; int bitrate = 20000; @@ -249,6 +250,15 @@ namespace moonlight_xbox_dx { } } + property bool VideoSuperResolution + { + bool get() { return this->videoSuperResolution; } + void set(bool value) { + this->videoSuperResolution = value; + OnPropertyChanged("VideoSuperResolution"); + } + } + property bool EnableHDR { bool get() { return this->enableHDR; } diff --git a/State/Stats.cpp b/State/Stats.cpp index c41027f4..46671c1d 100644 --- a/State/Stats.cpp +++ b/State/Stats.cpp @@ -3,6 +3,7 @@ #include "Utils.hpp" #include "../Plot/ImGuiPlots.h" #include "../Streaming/FFMpegDecoder.h" +#include "../State/StreamConfiguration.h" using namespace moonlight_xbox_dx; @@ -45,7 +46,12 @@ bool Stats::ShouldUpdateDisplay(DX::StepTimer const& timer, bool isVisible, char // Move this window into the last window slot and clear it for next window memcpy(&m_LastWndVideoStats, &m_ActiveWndVideoStats, sizeof(VIDEO_STATS)); + + // Preserve the scaleRatio which is set directly by the renderer and doesn't accumulate + float currentScaleRatio = m_ActiveWndVideoStats.scaleRatio; ZeroMemory(&m_ActiveWndVideoStats, sizeof(VIDEO_STATS)); + m_ActiveWndVideoStats.scaleRatio = currentScaleRatio; + m_ActiveWndVideoStats.measurementStartTimestamp = timer.GetTotalSeconds(); } @@ -149,6 +155,11 @@ void Stats::SubmitRenderStats(double preWaitTimeMs, double renderTimeMs, double m_ActiveWndVideoStats.totalPresentTimeUs += static_cast(presentTimeMs * 1000); } +void Stats::SubmitScaleRatio(float scaleRatio) { + std::lock_guard lock(m_mutex); + m_ActiveWndVideoStats.scaleRatio = scaleRatio; +} + /// private methods void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_STATS& dst) { @@ -167,6 +178,7 @@ void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_ST dst.totalPreWaitTimeUs += src.totalPreWaitTimeUs; dst.totalPresentTimeUs += src.totalPresentTimeUs; dst.totalPresentDisplayMs += src.totalPresentDisplayMs; + dst.scaleRatio = src.scaleRatio; if (dst.minHostProcessingLatency == 0) { dst.minHostProcessingLatency = src.minHostProcessingLatency; @@ -299,19 +311,39 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha double avgVideoMbps = m_bwTracker.GetAverageMbps(); double peakVideoMbps = m_bwTracker.GetPeakMbps(); - ret = snprintf(&output[offset], - length - offset, - "Bitrate: %.1f Mbps, Peak (%us): %.1f\n" - "Incoming frame rate from network: %.2f FPS\n" - "Decoding frame rate: %.2f FPS\n" - "Rendering frame rate: %.2f FPS (%s)\n", - avgVideoMbps, - m_bwTracker.GetWindowSeconds(), - peakVideoMbps, - stats.receivedFps, - stats.decodedFps, - stats.renderedFps, - Pacer::instance().getPacingImmediate() ? "immediate" : "display-locked"); + bool isVSREnabled = GetStreamConfig() != nullptr && GetStreamConfig()->videoSuperResolution; + + if (isVSREnabled) { + ret = snprintf(&output[offset], + length - offset, + "Bitrate: %.1f Mbps, Peak (%us): %.1f\n" + "Video Enhancement: (x%.2f) NIS Shader\n" + "Incoming frame rate from network: %.2f FPS\n" + "Decoding frame rate: %.2f FPS\n" + "Rendering frame rate: %.2f FPS (%s)\n", + avgVideoMbps, + m_bwTracker.GetWindowSeconds(), + peakVideoMbps, + stats.scaleRatio, + stats.receivedFps, + stats.decodedFps, + stats.renderedFps, + Pacer::instance().getPacingImmediate() ? "immediate" : "display-locked"); + } else { + ret = snprintf(&output[offset], + length - offset, + "Bitrate: %.1f Mbps, Peak (%us): %.1f\n" + "Incoming frame rate from network: %.2f FPS\n" + "Decoding frame rate: %.2f FPS\n" + "Rendering frame rate: %.2f FPS (%s)\n", + avgVideoMbps, + m_bwTracker.GetWindowSeconds(), + peakVideoMbps, + stats.receivedFps, + stats.decodedFps, + stats.renderedFps, + Pacer::instance().getPacingImmediate() ? "immediate" : "display-locked"); + } if (ret < 0 || (size_t)ret >= (length - offset)) { Utils::Log("Error: stringifyVideoStats length overflow\n"); return; diff --git a/State/Stats.h b/State/Stats.h index 7419e919..d2609ad3 100644 --- a/State/Stats.h +++ b/State/Stats.h @@ -46,6 +46,7 @@ typedef struct _VIDEO_STATS { double decodedFps; double renderedFps; double measurementStartTimestamp; + float scaleRatio; } VIDEO_STATS, *PVIDEO_STATS; namespace moonlight_xbox_dx @@ -64,6 +65,7 @@ namespace moonlight_xbox_dx void SubmitPacerTime(int64_t pacerTimeQpc); void SubmitPresentPacing(double presentDisplayMs); void SubmitRenderStats(double preWaitTimeMs, double renderTimeMs, double presentTimeMs, bool hitDeadline); + void SubmitScaleRatio(float scaleRatio); private: void addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_STATS& dst); diff --git a/State/StreamConfiguration.h b/State/StreamConfiguration.h index f0baa3c6..aa6887a6 100644 --- a/State/StreamConfiguration.h +++ b/State/StreamConfiguration.h @@ -17,6 +17,7 @@ namespace moonlight_xbox_dx property Platform::String^ framePacing; property bool enableHDR; property bool playAudioOnPC; + property bool videoSuperResolution; property bool enableVsync; property bool enableSOPS; property bool enableStats; diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index 4b9cb66d..8fd96c28 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -6,6 +6,7 @@ #include "../Plot/PlotDesc.h" #include "FFMpegDecoder.h" #include "Utils.hpp" +#include "../State/StreamConfiguration.h" using namespace DirectX; using namespace moonlight_xbox_dx; @@ -187,17 +188,19 @@ void StatsRenderer::CreateWindowSizeDependentResources() { int right = m_displayWidth / 3; int bottom = 0; - // 13 lines of text + // Check if Video Super Resolution is enabled to allocate space for 14 lines instead of 13 + bool isVSREnabled = GetStreamConfig() != nullptr && GetStreamConfig()->videoSuperResolution; + if (m_displayHeight >= 2160) { // 24pt font left = 20; right = m_displayWidth / 2; - bottom = 448; + bottom = isVSREnabled ? 483 : 448; // 14 lines (483) vs 13 lines (448) } else if (m_displayHeight >= 1440) { // 12pt font left = 14; - bottom = 224; + bottom = isVSREnabled ? 242 : 224; // 14 lines (242) vs 13 lines (224) } else { left = 10; - bottom = 224; + bottom = isVSREnabled ? 242 : 224; // 14 lines (242) vs 13 lines (224) } #if defined(_DEBUG) diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index 1f6c1e8d..13781876 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -118,9 +118,62 @@ bool VideoRenderer::Render(AVFrame *frame) { D3D11_TEXTURE2D_DESC ffmpegDesc; ffmpegTexture->GetDesc(&ffmpegDesc); + // First, calculate the true destination size and offsets (to preserve aspect ratio and add black bars) + IRECT src, fsrDst; + src.x = src.y = 0; + src.w = configuration->width; + src.h = configuration->height; + + // FSR true pixel-based destination rect + fsrDst.x = fsrDst.y = 0; + fsrDst.w = m_deviceResources->GetPixelWidth(); + fsrDst.h = m_deviceResources->GetPixelHeight(); + scaleSourceToDestinationSurface(&src, &fsrDst); + bool hasChanged = hasFrameFormatChanged(frame); if (hasChanged) { setupVideoTexture(ffmpegDesc); + + // Set the correct format depending on whether HDR is active + DXGI_FORMAT renderFormat = m_deviceResources->GetBackBufferFormat(); + bool isHDR = (frame->color_trc == AVCOL_TRC_SMPTE2084); + + // Initialize the Upscaler to upscale exactly to the aspect-ratio-preserved dimensions in raw pixels + if (configuration->videoSuperResolution) { + if (!m_upscaler) { + m_upscaler = std::make_unique(m_deviceResources); + } + if (m_upscaler->Initialize(src.w, src.h, fsrDst.w, fsrDst.h, renderFormat, isHDR)) { + D3D11_TEXTURE2D_DESC texDesc = {}; + texDesc.Width = src.w; + texDesc.Height = src.h; + texDesc.MipLevels = 1; + texDesc.ArraySize = 1; + texDesc.Format = renderFormat; + texDesc.SampleDesc.Count = 1; + texDesc.Usage = D3D11_USAGE_DEFAULT; + texDesc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + + auto device = m_deviceResources->GetD3DDevice(); + HRESULT hr = device->CreateTexture2D(&texDesc, nullptr, m_intermediateTex.ReleaseAndGetAddressOf()); + if (SUCCEEDED(hr)) { + hr = device->CreateRenderTargetView(m_intermediateTex.Get(), nullptr, m_intermediateRTV.ReleaseAndGetAddressOf()); + if (SUCCEEDED(hr)) { + hr = device->CreateShaderResourceView(m_intermediateTex.Get(), nullptr, m_intermediateSRV.ReleaseAndGetAddressOf()); + } + } + + if (FAILED(hr)) { + m_upscaler.reset(); + } + } else { + m_upscaler.reset(); + } + } + + // Calculate and submit scale ratio using the raw pixel output height / stream height + float scaleRatio = (float)fsrDst.h / (float)frame->height; + m_deviceResources->GetStats()->SubmitScaleRatio(scaleRatio); } // SRV 0 is always mapped to the video texture @@ -142,6 +195,31 @@ bool VideoRenderer::Render(AVFrame *frame) { bindColorConversion(frame, ffmpegDesc); } + // Bind the render target: If VSR is enabled AND successfully initialized, use intermediate RTV + if (configuration->videoSuperResolution && m_intermediateRTV && m_upscaler) { + // Render the YUV->RGB pass into our intermediate texture at native stream resolution + ID3D11RenderTargetView* renderTarget[] = { m_intermediateRTV.Get() }; + ctx->OMSetRenderTargets(1, renderTarget, nullptr); + + // Set viewport to match the intermediate texture dimensions + D3D11_VIEWPORT vp; + vp.Width = (float)configuration->width; + vp.Height = (float)configuration->height; + vp.MinDepth = 0.0f; + vp.MaxDepth = 1.0f; + vp.TopLeftX = 0; + vp.TopLeftY = 0; + ctx->RSSetViewports(1, &vp); + } else { + // Render directly to the screen (backbuffer) + ID3D11RenderTargetView* renderTarget[] = { m_deviceResources->GetBackBufferRenderTargetView() }; + ctx->OMSetRenderTargets(1, renderTarget, nullptr); + + // Set viewport to match the screen + auto screenViewport = m_deviceResources->GetScreenViewport(); + ctx->RSSetViewports(1, &screenViewport); + } + UINT stride = sizeof(VERTEX); UINT offset = 0; ctx->IASetVertexBuffers(0, 1, m_VideoVertexBuffer.GetAddressOf(), &stride, &offset); @@ -152,13 +230,45 @@ bool VideoRenderer::Render(AVFrame *frame) { ctx->PSSetShaderResources(0, 2, frameSrvs); ctx->PSSetConstantBuffers(0, 1, m_cscConstantBuffer.GetAddressOf()); - // Draw the video + // Draw the video (YUV to RGB conversion) ctx->DrawIndexed(6, 0, 0); // Unbind SRVs for this frame ID3D11ShaderResourceView* nullSrvs[2] = {}; ctx->PSSetShaderResources(0, 2, nullSrvs); + // If Video Super Resolution is enabled, run the compute shaders + if (configuration->videoSuperResolution && m_upscaler && m_intermediateSRV) { + // Unbind the Render Target so we can use its SRV in the Compute Shader without D3D11 Hazard Tracking forcing it to NULL + ID3D11RenderTargetView* nullRTV[1] = { nullptr }; + ctx->OMSetRenderTargets(1, nullRTV, nullptr); + + // Run EASU and RCAS passes on the intermediate texture + ID3D11ShaderResourceView* finalSRV = m_upscaler->ProcessFrame(m_intermediateSRV.Get()); + + if (finalSRV) { + // Now we need to render the final upscaled texture to the actual backbuffer + ID3D11RenderTargetView* backBufferRTV[] = { m_deviceResources->GetBackBufferRenderTargetView() }; + ctx->OMSetRenderTargets(1, backBufferRTV, nullptr); + + // For simplicity and to ensure it displays, if m_upscaler gives us an SRV, we can retrieve its texture + // and copy it to the backbuffer if dimensions match. + Microsoft::WRL::ComPtr upscaledRes; + finalSRV->GetResource(upscaledRes.ReleaseAndGetAddressOf()); + + Microsoft::WRL::ComPtr upscaledTex; + upscaledRes.As(&upscaledTex); + + if (upscaledTex) { + Microsoft::WRL::ComPtr backBufferRes; + backBufferRTV[0]->GetResource(backBufferRes.ReleaseAndGetAddressOf()); + + // Copy the upscaled texture to the backbuffer, using calculated offsets for centering + ctx->CopySubresourceRegion(backBufferRes.Get(), 0, fsrDst.x, fsrDst.y, 0, upscaledTex.Get(), 0, nullptr); + } + } + } + if (frame->color_trc != m_LastColorTrc) { DXGI_COLOR_SPACE_TYPE colorspace = {}; @@ -197,7 +307,7 @@ void VideoRenderer::CreateDeviceDependentResources() vertexShaderBytecode.data(), vertexShaderBytecode.size(), nullptr, - &m_vertexShader + m_vertexShader.ReleaseAndGetAddressOf() ) , "Vertex Shader Creation"); @@ -213,7 +323,7 @@ void VideoRenderer::CreateDeviceDependentResources() ARRAYSIZE(vertexDesc), vertexShaderBytecode.data(), vertexShaderBytecode.size(), - &m_inputLayout + m_inputLayout.ReleaseAndGetAddressOf() ) , "Input Layout Creation"); } @@ -226,7 +336,7 @@ void VideoRenderer::CreateDeviceDependentResources() pixelShaderBytecode.data(), pixelShaderBytecode.size(), nullptr, - &m_pixelShaderYUV420 + m_pixelShaderYUV420.ReleaseAndGetAddressOf() ) , "Pixel Shader Creation"); } @@ -257,7 +367,7 @@ void VideoRenderer::CreateDeviceDependentResources() samplerDesc.MinLOD = 0.0f; samplerDesc.MaxLOD = D3D11_FLOAT32_MAX; - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateSamplerState(&samplerDesc, &m_samplerState)); + DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateSamplerState(&samplerDesc, m_samplerState.ReleaseAndGetAddressOf())); } // We use a common index buffer for all geometry @@ -275,7 +385,7 @@ void VideoRenderer::CreateDeviceDependentResources() indexBufferData.pSysMem = indexes; indexBufferData.SysMemPitch = sizeof(int); - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateBuffer(&indexBufferDesc, &indexBufferData, &m_indexBuffer), "Index Buffer creation"); + DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateBuffer(&indexBufferDesc, &indexBufferData, m_indexBuffer.ReleaseAndGetAddressOf()), "Index Buffer creation"); } DISPATCH_THREADPOOL(([this, devRes = m_deviceResources, cfg = configuration] { @@ -306,6 +416,10 @@ void VideoRenderer::ReleaseDeviceDependentResources() m_VideoVertexBuffer.Reset(); m_samplerState.Reset(); m_indexBuffer.Reset(); + m_intermediateTex.Reset(); + m_intermediateRTV.Reset(); + m_intermediateSRV.Reset(); + m_upscaler.reset(); } void VideoRenderer::scaleSourceToDestinationSurface(IRECT* src, IRECT* dst) @@ -384,14 +498,24 @@ void VideoRenderer::setupVertexBuffer(D3D11_TEXTURE2D_DESC frameDesc) src.x = src.y = 0; src.w = configuration->width; src.h = configuration->height; - dst.x = dst.y = 0; - dst.w = m_DisplayWidth; - dst.h = m_DisplayHeight; - scaleSourceToDestinationSurface(&src, &dst); - - // Convert screen space to normalized device coordinates + FRECT renderRect; - screenSpaceToNormalizedDeviceCoords(&dst, &renderRect, m_DisplayWidth, m_DisplayHeight); + + if (configuration->videoSuperResolution) { + // Output exactly to the native resolution texture for the FSR pass + renderRect.x = -1.0f; + renderRect.y = -1.0f; + renderRect.w = 2.0f; + renderRect.h = 2.0f; + } else { + dst.x = dst.y = 0; + dst.w = m_DisplayWidth; + dst.h = m_DisplayHeight; + scaleSourceToDestinationSurface(&src, &dst); + + // Convert screen space to normalized device coordinates + screenSpaceToNormalizedDeviceCoords(&dst, &renderRect, m_DisplayWidth, m_DisplayHeight); + } m_TextureWidth = frameDesc.Width; m_TextureHeight = frameDesc.Height; diff --git a/Streaming/VideoRenderer.h b/Streaming/VideoRenderer.h index 48176378..1a7c20c5 100644 --- a/Streaming/VideoRenderer.h +++ b/Streaming/VideoRenderer.h @@ -5,6 +5,7 @@ #include "State\MoonlightClient.h" #include "State\StreamConfiguration.h" #include +#include "VideoUpscaler.h" extern "C" { #include @@ -97,6 +98,11 @@ namespace moonlight_xbox_dx AVColorTransferCharacteristic m_LastColorTrc = AVCOL_TRC_UNSPECIFIED; AVColorSpace m_LastColorSpace = AVCOL_SPC_UNSPECIFIED; AVChromaLocation m_LastChromaLocation = AVCHROMA_LOC_UNSPECIFIED; + + std::unique_ptr m_upscaler; + Microsoft::WRL::ComPtr m_intermediateTex; + Microsoft::WRL::ComPtr m_intermediateRTV; + Microsoft::WRL::ComPtr m_intermediateSRV; }; } diff --git a/Streaming/VideoUpscaler.cpp b/Streaming/VideoUpscaler.cpp new file mode 100644 index 00000000..aa5a6106 --- /dev/null +++ b/Streaming/VideoUpscaler.cpp @@ -0,0 +1,188 @@ +#include "pch.h" +#include "VideoUpscaler.h" +#include "..\Common\DirectXHelper.h" +#include "Utils.hpp" +#include + +using namespace moonlight_xbox_dx; + +VideoUpscaler::VideoUpscaler(const std::shared_ptr& deviceResources) + : m_deviceResources(deviceResources), m_outWidth(0), m_outHeight(0) { +} + +VideoUpscaler::~VideoUpscaler() { + Cleanup(); +} + +bool VideoUpscaler::CreateConstantBuffer(const void* data, size_t size, ID3D11Buffer** buffer) { + D3D11_BUFFER_DESC desc = {}; + desc.Usage = D3D11_USAGE_IMMUTABLE; + desc.ByteWidth = static_cast((size + 15) & ~15); + desc.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + + D3D11_SUBRESOURCE_DATA initData = {}; + initData.pSysMem = data; + + HRESULT hr = m_deviceResources->GetD3DDevice()->CreateBuffer(&desc, &initData, buffer); + return SUCCEEDED(hr); +} + +bool VideoUpscaler::CreateCoefTexture(const void* data, int width, int height, int rowPitch, DXGI_FORMAT format, ID3D11Texture2D** tex, ID3D11ShaderResourceView** srv) { + D3D11_TEXTURE2D_DESC desc = {}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = format; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + + D3D11_SUBRESOURCE_DATA initData = {}; + initData.pSysMem = data; + initData.SysMemPitch = rowPitch; + + auto device = m_deviceResources->GetD3DDevice(); + if (FAILED(device->CreateTexture2D(&desc, &initData, tex))) return false; + if (FAILED(device->CreateShaderResourceView(*tex, nullptr, srv))) return false; + return true; +} + +// Initialize NIS components: Load shaders, setup constants and allocate output textures. +bool VideoUpscaler::Initialize(int inWidth, int inHeight, int outWidth, int outHeight, DXGI_FORMAT format, bool isHDR) { + Cleanup(); + + if (outWidth == 0 || outHeight == 0 || inWidth == 0 || inHeight == 0) { + return false; + } + + m_outWidth = outWidth; + m_outHeight = outHeight; + + auto device = m_deviceResources->GetD3DDevice(); + + // Check if Half-Precision (FP16) is supported + D3D11_FEATURE_DATA_SHADER_MIN_PRECISION_SUPPORT featureSupport = {}; + device->CheckFeatureSupport(D3D11_FEATURE_SHADER_MIN_PRECISION_SUPPORT, &featureSupport, sizeof(featureSupport)); + bool supportsFP16 = (featureSupport.AllOtherShaderStagesMinPrecision & D3D11_SHADER_MIN_PRECISION_16_BIT) != 0; + + // Based on NIS static compilation parameters + UINT blockWidth = 32; + UINT blockHeight = supportsFP16 ? 32 : 24; + + std::wstring shaderPath = L"Assets\\Shader\\nis_"; + shaderPath += isHDR ? L"hdr_" : L"sdr_"; + shaderPath += supportsFP16 ? L"fp16.cso" : L"fp32.cso"; + + // 1. Load precompiled NIS shader + try { + auto shaderBlob = DX::ReadData(shaderPath.c_str()); + DX::ThrowIfFailed(device->CreateComputeShader(shaderBlob.data(), shaderBlob.size(), nullptr, m_nisShader.ReleaseAndGetAddressOf()), "Create NIS Compute Shader"); + } catch (Platform::Exception^ e) { + Utils::Logf("Failed to load NIS shader (%ls). HRESULT: 0x%X\n", shaderPath.c_str(), e->HResult); + return false; + } catch (...) { + Utils::Log("Failed to load NIS shader with unknown exception\n"); + return false; + } + + // 2. Initialize NIS configuration constants + NISConfig config = {}; + float sharpness = 0.25f; + NISHDRMode hdrMode = isHDR ? NISHDRMode::PQ : NISHDRMode::None; + + NVScalerUpdateConfig(config, sharpness, 0, 0, inWidth, inHeight, inWidth, inHeight, 0, 0, outWidth, outHeight, outWidth, outHeight, hdrMode); + + if (!CreateConstantBuffer(&config, sizeof(config), m_nisCB.ReleaseAndGetAddressOf())) return false; + + // 3. Create coefficient textures + const int rowPitch = supportsFP16 ? kFilterSize * sizeof(uint16_t) : kFilterSize * sizeof(float); + DXGI_FORMAT coefFormat = supportsFP16 ? DXGI_FORMAT_R16G16B16A16_FLOAT : DXGI_FORMAT_R32G32B32A32_FLOAT; + const void* scaleData = supportsFP16 ? (const void*)coef_scale_fp16 : (const void*)coef_scale; + const void* usmData = supportsFP16 ? (const void*)coef_usm_fp16 : (const void*)coef_usm; + + if (!CreateCoefTexture(scaleData, kFilterSize / 4, kPhaseCount, rowPitch, coefFormat, m_coefScaleTex.ReleaseAndGetAddressOf(), m_coefScaleSRV.ReleaseAndGetAddressOf())) return false; + if (!CreateCoefTexture(usmData, kFilterSize / 4, kPhaseCount, rowPitch, coefFormat, m_coefUsmTex.ReleaseAndGetAddressOf(), m_coefUsmSRV.ReleaseAndGetAddressOf())) return false; + + Utils::Logf("VideoUpscaler: Initialized NIS (%dx%d -> %dx%d, HDR: %d, FP16: %d, Block: %ux%u)\n", inWidth, inHeight, outWidth, outHeight, isHDR, supportsFP16, blockWidth, blockHeight); + + // 4. Create output texture (supporting UAV and SRV) + D3D11_TEXTURE2D_DESC texDesc = {}; + texDesc.Width = outWidth; + texDesc.Height = outHeight; + texDesc.MipLevels = 1; + texDesc.ArraySize = 1; + texDesc.Format = format; + texDesc.SampleDesc.Count = 1; + texDesc.Usage = D3D11_USAGE_DEFAULT; + texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_UNORDERED_ACCESS; + + HRESULT hr = device->CreateTexture2D(&texDesc, nullptr, m_finalOutputTex.ReleaseAndGetAddressOf()); + if (FAILED(hr)) return false; + + hr = device->CreateShaderResourceView(m_finalOutputTex.Get(), nullptr, m_finalOutputSRV.ReleaseAndGetAddressOf()); + if (FAILED(hr)) return false; + + hr = device->CreateUnorderedAccessView(m_finalOutputTex.Get(), nullptr, m_finalOutputUAV.ReleaseAndGetAddressOf()); + if (FAILED(hr)) { + Utils::Logf("Failed to create UAV for NIS final texture. HRESULT: 0x%X\n", hr); + return false; + } + + // 5. Create Sampler (Linear Clamp) + D3D11_SAMPLER_DESC sampDesc = {}; + sampDesc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR; + sampDesc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; + sampDesc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP; + sampDesc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + hr = device->CreateSamplerState(&sampDesc, m_sampler.ReleaseAndGetAddressOf()); + if (FAILED(hr)) return false; + + // 6. Calculate Dispatch arguments + m_dispatchX = static_cast(std::ceil(m_outWidth / float(blockWidth))); + m_dispatchY = static_cast(std::ceil(m_outHeight / float(blockHeight))); + + return true; +} + +// Dispatches the compute shader to apply NIS to the input frame. +ID3D11ShaderResourceView* VideoUpscaler::ProcessFrame(ID3D11ShaderResourceView* inputSRV) { + if (!m_nisShader || !inputSRV) return inputSRV; + + auto ctx = m_deviceResources->GetD3DDeviceContext(); + + ID3D11SamplerState* samplers[] = { m_sampler.Get() }; + ctx->CSSetSamplers(0, 1, samplers); + + // NIS Pass (Upscale + Sharpen) + ctx->CSSetShader(m_nisShader.Get(), nullptr, 0); + ctx->CSSetConstantBuffers(0, 1, m_nisCB.GetAddressOf()); + + ID3D11ShaderResourceView* srvs[] = { inputSRV, m_coefScaleSRV.Get(), m_coefUsmSRV.Get() }; + ctx->CSSetShaderResources(0, 3, srvs); + ctx->CSSetUnorderedAccessViews(0, 1, m_finalOutputUAV.GetAddressOf(), nullptr); + + ctx->Dispatch(m_dispatchX, m_dispatchY, 1); + + // Clean up CS state + ID3D11UnorderedAccessView* nullUAV[] = { nullptr }; + ctx->CSSetUnorderedAccessViews(0, 1, nullUAV, nullptr); + ID3D11ShaderResourceView* nullSRVs[] = { nullptr, nullptr, nullptr }; + ctx->CSSetShaderResources(0, 3, nullSRVs); + + return m_finalOutputSRV.Get(); +} + +// Release all associated resources to prevent leaks. +void VideoUpscaler::Cleanup() { + m_nisShader.Reset(); + m_nisCB.Reset(); + m_coefScaleTex.Reset(); + m_coefScaleSRV.Reset(); + m_coefUsmTex.Reset(); + m_coefUsmSRV.Reset(); + m_finalOutputTex.Reset(); + m_finalOutputSRV.Reset(); + m_finalOutputUAV.Reset(); + m_sampler.Reset(); +} diff --git a/Streaming/VideoUpscaler.h b/Streaming/VideoUpscaler.h new file mode 100644 index 00000000..5e7ad251 --- /dev/null +++ b/Streaming/VideoUpscaler.h @@ -0,0 +1,56 @@ +#pragma once + +#include "pch.h" +#include +#include +#include +#include "..\Common\DeviceResources.h" + +// Define NIS generic architecture mapping based on GPU +#include + +namespace moonlight_xbox_dx { + + class VideoUpscaler { + public: + VideoUpscaler(const std::shared_ptr& deviceResources); + ~VideoUpscaler(); + + // Initialize the shaders, buffers, and textures for a fixed input/output resolution + bool Initialize(int inWidth, int inHeight, int outWidth, int outHeight, DXGI_FORMAT format, bool isHDR); + + // Processes the input SRV through NIS scaling and sharpening pass + ID3D11ShaderResourceView* ProcessFrame(ID3D11ShaderResourceView* inputSRV); + + // Cleans up all D3D resources + void Cleanup(); + + private: + std::shared_ptr m_deviceResources; + + Microsoft::WRL::ComPtr m_nisShader; + + Microsoft::WRL::ComPtr m_nisCB; + + // NIS Coefficient textures + Microsoft::WRL::ComPtr m_coefScaleTex; + Microsoft::WRL::ComPtr m_coefScaleSRV; + Microsoft::WRL::ComPtr m_coefUsmTex; + Microsoft::WRL::ComPtr m_coefUsmSRV; + + Microsoft::WRL::ComPtr m_finalOutputTex; + Microsoft::WRL::ComPtr m_finalOutputSRV; + Microsoft::WRL::ComPtr m_finalOutputUAV; + + Microsoft::WRL::ComPtr m_sampler; + + int m_outWidth; + int m_outHeight; + uint32_t m_dispatchX; + uint32_t m_dispatchY; + + // Helper to initialize constant buffers + bool CreateConstantBuffer(const void* data, size_t size, ID3D11Buffer** buffer); + bool CreateCoefTexture(const void* data, int width, int height, int rowPitch, DXGI_FORMAT format, ID3D11Texture2D** tex, ID3D11ShaderResourceView** srv); + }; +} diff --git a/moonlight-xbox-dx.vcxproj b/moonlight-xbox-dx.vcxproj index 82c7f3d7..34298a3f 100644 --- a/moonlight-xbox-dx.vcxproj +++ b/moonlight-xbox-dx.vcxproj @@ -1,4 +1,4 @@ - + @@ -146,7 +146,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 _DEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -162,7 +162,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 NDEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -178,7 +178,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 _DEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -194,7 +194,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 NDEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -210,7 +210,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 _DEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -226,7 +226,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(IntermediateOutputPath);%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 NDEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -242,7 +242,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(ProjectDir)vcpkg_installed\x64-uwp\include;$(IntermediateOutputPath);third_party\moonlight-common-c\src;third_party\imgui;third_party\imgui\backends;third_party\imgui-uwp\backends;third_party\implot;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(ProjectDir)vcpkg_installed\x64-uwp\include;$(IntermediateOutputPath);third_party\moonlight-common-c\src;third_party\imgui;third_party\imgui\backends;third_party\imgui-uwp\backends;third_party\implot;%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -262,7 +262,7 @@ pch.h $(IntDir)pch.pch - $(ProjectDir);$(SolutionDir)\third_party\DirectXTK\inc;$(ProjectDir)vcpkg_installed\x64-uwp\include;$(IntermediateOutputPath);third_party\moonlight-common-c\src;third_party\imgui;third_party\imgui\backends;third_party\imgui-uwp\backends;third_party\implot;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)third_party\NVIDIAImageScaling\NIS;$(SolutionDir)\third_party\DirectXTK\inc;$(ProjectDir)vcpkg_installed\x64-uwp\include;$(IntermediateOutputPath);third_party\moonlight-common-c\src;third_party\imgui;third_party\imgui\backends;third_party\imgui-uwp\backends;third_party\implot;%(AdditionalIncludeDirectories) /bigobj %(AdditionalOptions) 4453;28204 _CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;NDEBUG;%(PreprocessorDefinitions);_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING @@ -354,6 +354,7 @@ + @@ -491,6 +492,7 @@ + @@ -616,6 +618,25 @@ true + + + + + true + Document + + + true + Document + + + true + Document + + + true + Document + true Document diff --git a/moonlight-xbox-dx.vcxproj.filters b/moonlight-xbox-dx.vcxproj.filters index 6e8d645d..d9592e37 100644 --- a/moonlight-xbox-dx.vcxproj.filters +++ b/moonlight-xbox-dx.vcxproj.filters @@ -1,4 +1,4 @@ - + @@ -45,6 +45,9 @@ Source Files + + Source Files + Source Files @@ -381,6 +384,9 @@ Source Files + + Source Files + @@ -401,6 +407,9 @@ Header Files + + Header Files + Header Files @@ -476,6 +485,9 @@ Header Files + + Header Files + @@ -672,8 +684,29 @@ Assets + + Shaders + + + Shaders + + + Shaders + + + Shaders + + + Shaders + + + Header Files + + + Header Files + diff --git a/third_party/NVIDIAImageScaling b/third_party/NVIDIAImageScaling new file mode 160000 index 00000000..35e13ba3 --- /dev/null +++ b/third_party/NVIDIAImageScaling @@ -0,0 +1 @@ +Subproject commit 35e13ba316c98eeecf16f37eae70ce88019911f6