Add Video Super Resolution for Windows x64 (AMD, Intel and NVIDIA), Windows ARM (Snapdragon), Linux and MacOS

.github/workflows/build.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     branches:
       - master
+  workflow_dispatch:
 
 concurrency:
   group: build-${{ github.ref }}

.gitignore

@@ -1,9 +1,32 @@
 *pro.user
-
+.qtcreator/
 
 **/.vs/
 .vscode/
 build/
 libs/
 config.tests/*/.qmake.stash
 config.tests/*/Makefile
+
+third-party/**
+!third-party/
+
+# Submodules
+!third-party/AMF/
+!third-party/DirectX-Headers/
+!third-party/FidelityFX-FSR/
+!third-party/NVIDIAImageScaling/
+!third-party/libvpl/
+
+# Bundled SDKs
+!third-party/DirectXShaderCompiler/
+!third-party/DirectXShaderCompiler/**
+!third-party/RTX_Video_SDK/
+!third-party/RTX_Video_SDK/**
+!third-party/IntelVPL/
+!third-party/IntelVPL/**/
+!third-party/IntelVPL/*/lib/**
+!third-party/IntelVPL/*/include/**
+!third-party/IntelVPL/*/bin/libvpl.dll
+!third-party/stb_image/
+!third-party/stb_image/**

.gitmodules

@@ -1,9 +1,32 @@
 [submodule "moonlight-common-c/moonlight-common-c"]
 	path = moonlight-common-c/moonlight-common-c
 	url = https://github.com/moonlight-stream/moonlight-common-c.git
+	ignore = dirty
 [submodule "qmdnsengine/qmdnsengine"]
 	path = qmdnsengine/qmdnsengine
 	url = https://github.com/cgutman/qmdnsengine.git
+	ignore = dirty
 [submodule "app/SDL_GameControllerDB"]
 	path = app/SDL_GameControllerDB
 	url = https://github.com/gabomdq/SDL_GameControllerDB.git
+	ignore = dirty
+[submodule "third-party/AMF"]
+	path = third-party/AMF
+	url = https://github.com/GPUOpen-LibrariesAndSDKs/AMF.git
+	ignore = dirty
+[submodule "third-party/FidelityFX-FSR"]
+	path = third-party/FidelityFX-FSR
+	url = https://github.com/GPUOpen-Effects/FidelityFX-FSR.git
+	ignore = dirty
+[submodule "third-party/NVIDIAImageScaling"]
+	path = third-party/NVIDIAImageScaling
+	url = https://github.com/NVIDIAGameWorks/NVIDIAImageScaling.git
+	ignore = dirty
+[submodule "third-party/DirectX-Headers"]
+	path = third-party/DirectX-Headers
+	url = https://github.com/microsoft/DirectX-Headers.git
+	ignore = dirty
+[submodule "third-party/libvpl"]
+	path = third-party/libvpl
+	url = https://github.com/intel/libvpl.git
+	ignore = dirty

app/app.pro

@@ -1,4 +1,4 @@
-QT += core quick network quickcontrols2 svg
+QT += core quick network quickcontrols2 svg concurrent
 CONFIG += c++17
 
 unix:!macx {
@@ -149,7 +149,7 @@ unix:if(!macx|disable-prebuilts) {
     }
 }
 win32 {
-    LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lswscale -lopus -ldxgi -ld3d11 -llibplacebo
+    LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lswscale -lopus -ldxgi -ld3d11 -ld3d12 -ldxguid -llibplacebo -ld3dcompiler -ldxcompiler
     CONFIG += ffmpeg libplacebo
 }
 win32:!winrt {
@@ -161,7 +161,7 @@ macx {
         CONFIG += discord-rpc libplacebo
     }
 
-    LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreGraphics -framework CoreMedia -framework AppKit -framework Metal -framework QuartzCore
+    LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreGraphics -framework CoreMedia -framework AppKit -framework Metal -framework MetalFx -framework MetalPerformanceShaders -framework QuartzCore
     CONFIG += ffmpeg
 }
 
@@ -204,6 +204,7 @@ SOURCES += \
     gui/sdlgamepadkeynavigation.cpp \
     streaming/video/overlaymanager.cpp \
     backend/systemproperties.cpp \
+    streaming/video/videoenhancement.cpp \
     wm.cpp
 
 HEADERS += \
@@ -241,7 +242,8 @@ HEADERS += \
     settings/mappingmanager.h \
     gui/sdlgamepadkeynavigation.h \
     streaming/video/overlaymanager.h \
-    backend/systemproperties.h
+    backend/systemproperties.h \
+    streaming/video/videoenhancement.h
 
 # Platform-specific renderers and decoders
 ffmpeg {
@@ -391,18 +393,148 @@ config_SL {
 win32 {
     HEADERS += streaming/video/ffmpeg-renderers/dxutil.h
 }
+
+win32:!winrt {
+    message(NVIDIA VSR and TrueHDR technologies)
+
+    # Required by NVIDIA RTX Video SDK; compilation fails without these linker flags
+    # as the SDK is compiled to work with Visual Studio initially
+    # Note: '/guard:ehcont,no' disables some EH checks
+    # For a standalone application without external modules, this poses virtually no risk
+    QMAKE_LFLAGS += /guard:ehcont,no
+
+    LIBS += -ladvapi32
+
+    OS_ARCHI = x64
+    contains(QT_ARCH, arm64) {
+        OS_ARCHI = arm64
+    }
+
+    NGX_DLL_PATH_VSR = "$$PWD/../third-party/RTX_Video_SDK/bin/Windows/$${OS_ARCHI}/rel/nvngx_vsr.dll"
+    NGX_DLL_PATH_HDR = "$$PWD/../third-party/RTX_Video_SDK/bin/Windows/$${OS_ARCHI}/rel/nvngx_truehdr.dll"
+
+    CONFIG(debug, debug|release) {
+        # Debug
+        copy_vsr.commands = $$quote(copy /Y $$shell_path($$NGX_DLL_PATH_VSR) $$shell_path("$$OUT_PWD/debug"))
+        copy_hdr.commands = $$quote(copy /Y $$shell_path($$NGX_DLL_PATH_HDR) $$shell_path("$$OUT_PWD/debug"))
+        LIBS += -L$$PWD/../third-party/RTX_Video_SDK/lib/Windows/$${OS_ARCHI} -lnvsdk_ngx_d_dbg
+    }
+
+    CONFIG(release, debug|release) {
+        # Release
+        copy_vsr.commands = $$quote(copy /Y $$shell_path($$NGX_DLL_PATH_VSR) $$shell_path("$$OUT_PWD/release"))
+        copy_hdr.commands = $$quote(copy /Y $$shell_path($$NGX_DLL_PATH_HDR) $$shell_path("$$OUT_PWD/release"))
+        LIBS += -L$$PWD/../third-party/RTX_Video_SDK/lib/Windows/$${OS_ARCHI} -lnvsdk_ngx_d
+    }
+
+    QMAKE_POST_LINK += $$copy_vsr.commands & $$copy_hdr.commands
+
+    INCLUDEPATH += $$PWD/../third-party/RTX_Video_SDK/include
+}
+
+win32:!winrt {
+    message(AMD Upscaling technologies)
+
+    SOURCES += \
+        ../third-party/AMF/amf/public/common/AMFFactory.cpp \
+        ../third-party/AMF/amf/public/common/AMFSTL.cpp \
+        ../third-party/AMF/amf/public/common/Thread.cpp \
+        ../third-party/AMF/amf/public/common/TraceAdapter.cpp \
+        ../third-party/AMF/amf/public/common/Windows/ThreadWindows.cpp
+
+    INCLUDEPATH +=  \
+        $$PWD/../third-party/AMF/amf \
+        $$PWD/shaders/enhancer/AMD
+}
+
 win32:!winrt {
-    message(DXVA2 and D3D11VA renderers selected)
+    message(Intel VPL Upscaling technologies)
+
+    # IntelVPL already exists and is compiled for x64 architecture.
+    # Uncomment if you need to rebuild IntelVPL locally.
+    # Important: MSVC Build Tools for ARM64 and WinGet (https://github.com/microsoft/winget-cli/releases) are required,
+
+    # # Compile x64 architecture
+    # vpl_build_x64.commands = \
+    #     cd $$shell_path($$PWD/../third-party/libvpl) && \
+    #     script\\bootstrap.bat && \
+    #     cmake -B _build_x64 -DCMAKE_INSTALL_PREFIX="$$PWD/../third-party/IntelVPL/x64" && \
+    #     cmake --build _build_x64 --config Release && \
+    #     cmake --install _build_x64 --config Release
+    # vpl_build_x64.CONFIG += no_link
+    # QMAKE_EXTRA_TARGETS += vpl_build_x64
+    # PRE_TARGETDEPS += vpl_build_x64
+
+    # # Compile arm64 architecture
+    # # Note: Intel does not provide a arm64 version of its library,
+    # # but we need to compile it to avoid a build error, even if it is not used
+    # vpl_build_arm64.commands = \
+    #     cd $$shell_path($$PWD/../third-party/libvpl) && \
+    #     script\\bootstrap.bat && \
+    #     cmake -B _build_arm64 -A ARM64 -T host=x64 -DCMAKE_INSTALL_PREFIX="$$PWD/../third-party/IntelVPL/arm64" && \
+    #     cmake --build _build_arm64 --config Release && \
+    #     cmake --install _build_arm64 --config Release
+    # vpl_build_arm64.CONFIG += no_link
+    # QMAKE_EXTRA_TARGETS += vpl_build_arm64
+    # PRE_TARGETDEPS += vpl_build_arm64
+
+    OS_ARCHI = x64
+    contains(QT_ARCH, arm64) {
+        OS_ARCHI = arm64
+    }
+
+    INCLUDEPATH += $$PWD/../third-party/IntelVPL/$${OS_ARCHI}/include
+    LIBS += -L$$PWD/../third-party/IntelVPL/$${OS_ARCHI}/lib -lvpl
+
+    VPL_DLL_PATH = "$$PWD/../third-party/IntelVPL/$${OS_ARCHI}/bin/libvpl.dll"
+
+    CONFIG(debug, debug|release) {
+        # Debug
+        copy_vpl.commands = $$quote(copy /Y $$shell_path($$VPL_DLL_PATH) $$shell_path("$$OUT_PWD/debug"))
+    }
+
+    CONFIG(release, debug|release) {
+        # Release
+        copy_vpl.commands = $$quote(copy /Y $$shell_path($$VPL_DLL_PATH) $$shell_path("$$OUT_PWD/release"))
+    }
+
+    QMAKE_POST_LINK += & $$copy_vpl.commands
+
+    # IntelVPL is only available for the architecture x64
+    contains(QT_ARCH, x86_64) {
+        DEFINES += HAVE_INTEL_VPL
+    }
+}
+win32:!winrt {
+    message(NVIDIA Image Scaling)
+
+    INCLUDEPATH += $$PWD/../third-party/NVIDIAImageScaling/NIS
+}
+win32:!winrt {
+    message(Direct3D 12 headers)
+
+    INCLUDEPATH += $$PWD/../third-party/DirectX-Headers/include
+}
+win32:!winrt {
+    message("DXVA2, D3D11renderers and D3D12renderers selected")
 
     SOURCES += \
         streaming/video/ffmpeg-renderers/dxva2.cpp \
         streaming/video/ffmpeg-renderers/d3d11va.cpp \
+        streaming/video/ffmpeg-renderers/d3d12va.cpp \
+        streaming/video/ffmpeg-renderers/d3d12va_shaders.cpp \
         streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.cpp
 
     HEADERS += \
         streaming/video/ffmpeg-renderers/dxva2.h \
         streaming/video/ffmpeg-renderers/d3d11va.h \
+        streaming/video/ffmpeg-renderers/d3d12va.h \
+        streaming/video/ffmpeg-renderers/d3d12va_shaders.h \
         streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.h
+
+    CONFIG(debug, debug|release) {
+        INCLUDEPATH += $$PWD/../third-party/stb_image
+    }
 }
 macx {
     message(VideoToolbox renderer selected)

app/backend/systemproperties.cpp

@@ -6,6 +6,7 @@
 
 #include "streaming/session.h"
 #include "streaming/streamutils.h"
+#include "streaming/video/videoenhancement.h"
 
 #ifdef Q_OS_WIN32
 #define WIN32_LEAN_AND_MEAN
@@ -277,3 +278,38 @@ void SystemProperties::refreshDisplays()
 
     SDL_QuitSubSystem(SDL_INIT_VIDEO);
 }
+
+/**
+ * \brief Inform if the menu is selectable (DEBUG mode)
+ *
+ * For Debugging purpose only, we allow the dropdown menu visible to test other algorithms
+ *
+ * \return bool Returns true if in debug mode
+ */
+bool SystemProperties::isVideoEnhancementSwitchable()
+{
+#if defined(QT_DEBUG) && defined(Q_OS_WIN)
+    return true;
+#else
+    return false;
+#endif
+}
+
+/**
+ * \brief Inform if the GPU is capable of Video enhancement
+ *
+ * Check if either Video Super-Resolution features can be used by the GPU.
+ *
+ * isAvailable() only reflects that FFmpeg advertises a hwaccel able to carry the
+ * feature, which stays true on systems where the D3D12 device cannot actually be
+ * created. On Windows the upscaler lives in the D3D12 renderer (the Vulkan/FSR1
+ * path is not user-selectable there), so a D3D12 failure means the feature is out
+ * of reach and the setting must be greyed out.
+ *
+ * \return bool Returns true if the GPU is capable
+ */
+bool SystemProperties::isVideoEnhancementAvailable()
+{
+    return VideoEnhancement::getInstance().isAvailable() &&
+           VideoEnhancement::getInstance().isD3D12Available();
+}

app/backend/systemproperties.h

@@ -38,6 +38,8 @@ class SystemProperties : public QObject
     Q_INVOKABLE QRect getNativeResolution(int displayIndex);
     Q_INVOKABLE QRect getSafeAreaResolution(int displayIndex);
     Q_INVOKABLE int getRefreshRate(int displayIndex);
+    Q_INVOKABLE bool isVideoEnhancementSwitchable();
+    Q_INVOKABLE bool isVideoEnhancementAvailable();
 
     Q_INVOKABLE void startAsyncLoad();
     Q_INVOKABLE void waitForAsyncLoad();

app/cli/commandlineparser.cpp

@@ -305,6 +305,16 @@ StreamCommandLineParser::StreamCommandLineParser()
         {"5.1-surround", StreamingPreferences::AC_51_SURROUND},
         {"7.1-surround", StreamingPreferences::AC_71_SURROUND},
     };
+    m_SuperResolutionModeMap = {
+        {"auto",                       StreamingPreferences::SRM_00},
+        {"driver",                     StreamingPreferences::SRM_01},
+        {"video-processor",            StreamingPreferences::SRM_02},
+        {"fsr1-upscaler",              StreamingPreferences::SRM_03},
+        {"nis-upscaler",               StreamingPreferences::SRM_04},
+        {"sgsr1-upscaler",             StreamingPreferences::SRM_05},
+        {"rcas-sharpener",             StreamingPreferences::SRM_06},
+        {"nis-sharpener",              StreamingPreferences::SRM_07},
+    };
     m_VideoCodecMap = {
         {"auto",  StreamingPreferences::VCC_AUTO},
         {"H.264", StreamingPreferences::VCC_FORCE_H264},
@@ -352,6 +362,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     parser.addValueOption("packet-size", "video packet size");
     parser.addChoiceOption("display-mode", "display mode", m_WindowModeMap.keys());
     parser.addChoiceOption("audio-config", "audio config", m_AudioConfigMap.keys());
+    parser.addChoiceOption("super-resolution-mode", "super resolution mode", m_SuperResolutionModeMap.keys());
     parser.addToggleOption("multi-controller", "multiple controller support");
     parser.addToggleOption("quit-after", "quit app after session");
     parser.addToggleOption("absolute-mouse", "remote desktop optimized mouse control");
@@ -360,6 +371,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     parser.addToggleOption("game-optimization", "game optimizations");
     parser.addToggleOption("audio-on-host", "audio on host PC");
     parser.addToggleOption("frame-pacing", "frame pacing");
+    parser.addToggleOption("video-enhancement", "Enhance video with AI");
     parser.addToggleOption("mute-on-focus-loss", "mute audio when Moonlight window loses focus");
     parser.addToggleOption("background-gamepad", "background gamepad input");
     parser.addToggleOption("reverse-scroll-direction", "inverted scroll direction");
@@ -443,6 +455,11 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
         preferences->audioConfig = mapValue(m_AudioConfigMap, parser.getChoiceOptionValue("audio-config"));
     }
 
+    // Resolve --super-resolution-config option
+    if (parser.isSet("super-resolution-mode")) {
+        preferences->superResolutionMode = mapValue(m_SuperResolutionModeMap, parser.getChoiceOptionValue("super-resolution-mode"));
+    }
+
     // Resolve --multi-controller and --no-multi-controller options
     preferences->multiController = parser.getToggleOptionValue("multi-controller", preferences->multiController);
 
@@ -467,6 +484,9 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     // Resolve --frame-pacing and --no-frame-pacing options
     preferences->framePacing = parser.getToggleOptionValue("frame-pacing", preferences->framePacing);
 
+    // Resolve --video-enhancement and --no-video-enhancement options
+    preferences->videoEnhancing = parser.getToggleOptionValue("video-enhancement", preferences->videoEnhancing);
+
     // Resolve --mute-on-focus-loss and --no-mute-on-focus-loss options
     preferences->muteOnFocusLoss = parser.getToggleOptionValue("mute-on-focus-loss", preferences->muteOnFocusLoss);
 

app/cli/commandlineparser.h

@@ -69,6 +69,7 @@ class StreamCommandLineParser
     QString m_AppName;
     QMap<QString, StreamingPreferences::WindowMode> m_WindowModeMap;
     QMap<QString, StreamingPreferences::AudioConfig> m_AudioConfigMap;
+    QMap<QString, StreamingPreferences::SuperResolutionMode> m_SuperResolutionModeMap;
     QMap<QString, StreamingPreferences::VideoCodecConfig> m_VideoCodecMap;
     QMap<QString, StreamingPreferences::VideoDecoderSelection> m_VideoDecoderMap;
     QMap<QString, StreamingPreferences::CaptureSysKeysMode> m_CaptureSysKeysModeMap;