Add Video Super Resolution for Windows (AMD, Intel and NVIDIA) and MacOS

Video Super Resolution (VSR) is to Video as DLSS is to 3D Rendering.
Why not let Moonlight being one of the first game streaming solution leveraging such technology?
AI upscaling means significantly less bandwidth usage without compromising the video quality!

Tested with these GPUs:
Nvidia RTX 4070 Ti (Windows)
AMD RX 7600 (Windows)
AMD iGPU 780M (Windows)
Intel Arc A380 (Windows)
Intel UHD Graphics (16EU), an iGPU from N95 CPU (Windows)
M1 Pro (MacOS)
M3 Pro (MacOS)

Author: linckosz

.gitignore

@@ -2,6 +2,7 @@
 
 
 **/.vs/
+**/.vscode/
 build/
 config.tests/*/.qmake.stash
 config.tests/*/Makefile

.gitmodules

@@ -1,19 +1,22 @@
 [submodule "moonlight-common-c/moonlight-common-c"]
-	path = moonlight-common-c/moonlight-common-c
+        path = moonlight-common-c/moonlight-common-c
 	url = https://github.com/moonlight-stream/moonlight-common-c.git
 [submodule "qmdnsengine/qmdnsengine"]
-	path = qmdnsengine/qmdnsengine
+        path = qmdnsengine/qmdnsengine
 	url = https://github.com/cgutman/qmdnsengine.git
 [submodule "app/SDL_GameControllerDB"]
-	path = app/SDL_GameControllerDB
+        path = app/SDL_GameControllerDB
 	url = https://github.com/gabomdq/SDL_GameControllerDB.git
 [submodule "soundio/libsoundio"]
-	path = soundio/libsoundio
+        path = soundio/libsoundio
 	url = https://github.com/cgutman/libsoundio.git
 [submodule "h264bitstream/h264bitstream"]
-	path = h264bitstream/h264bitstream
+        path = h264bitstream/h264bitstream
 	url = https://github.com/aizvorski/h264bitstream.git
 [submodule "libs"]
-	path = libs
+        path = libs
 	url = https://github.com/cgutman/moonlight-qt-prebuilts.git
 	shallow = true
+[submodule "third-party/AMF"]
+        path = third-party/AMF
+	url = https://github.com/GPUOpen-LibrariesAndSDKs/AMF.git

app/app.pro

@@ -165,7 +165,7 @@ macx {
         CONFIG += discord-rpc
     }
 
-    LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreGraphics -framework CoreMedia -framework AppKit -framework Metal -framework QuartzCore
+    LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreGraphics -framework CoreMedia -framework AppKit -framework Metal -framework MetalFx -framework QuartzCore
 
     # For libsoundio
     LIBS += -framework CoreAudio -framework AudioUnit
@@ -211,6 +211,7 @@ SOURCES += \
     gui/sdlgamepadkeynavigation.cpp \
     streaming/video/overlaymanager.cpp \
     backend/systemproperties.cpp \
+    streaming/video/videoenhancement.cpp \
     wm.cpp
 
 HEADERS += \
@@ -220,6 +221,7 @@ HEADERS += \
     cli/pair.h \
     settings/compatfetcher.h \
     settings/mappingfetcher.h \
+    streaming/video/videoenhancement.h \
     utils.h \
     backend/computerseeker.h \
     backend/identitymanager.h \
@@ -403,6 +405,18 @@ win32:!winrt {
         streaming/video/ffmpeg-renderers/d3d11va.h \
         streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.h
 }
+win32:!winrt {
+    message(AMF enabled for AMD Drivers)
+
+    SOURCES += \
+        ../third-party/AMF/amf/public/common/AMFFactory.cpp \
+        ../third-party/AMF/amf/public/common/AMFSTL.cpp \
+        ../third-party/AMF/amf/public/common/Thread.cpp \
+        ../third-party/AMF/amf/public/common/TraceAdapter.cpp \
+        ../third-party/AMF/amf/public/common/Windows/ThreadWindows.cpp
+
+    INCLUDEPATH += $$PWD/../third-party/AMF/amf
+}
 macx {
     message(VideoToolbox renderer selected)
 

app/backend/systemproperties.cpp

@@ -6,6 +6,7 @@
 
 #include "streaming/session.h"
 #include "streaming/streamutils.h"
+#include "streaming/video/videoenhancement.h"
 
 #ifdef Q_OS_WIN32
 #define WIN32_LEAN_AND_MEAN
@@ -248,3 +249,27 @@ void SystemProperties::refreshDisplaysInternal()
 
     SDL_QuitSubSystem(SDL_INIT_VIDEO);
 }
+
+/**
+ * \brief Inform if the GPU is capable of Video enhancement
+ *
+ * Check if either Video Super-Resolution or SDR-to-HDR features can be used by the GPU.
+ *
+ * \return bool Returns true if the GPU is capable
+ */
+bool SystemProperties::isVideoEnhancementCapable()
+{
+    VideoEnhancement* videoEnhancement = &VideoEnhancement::getInstance();
+    return videoEnhancement->isUIvisible() && (videoEnhancement->isVSRcapable() || videoEnhancement->isHDRcapable());
+}
+
+/**
+ * \brief Inform if the GPU's driver is at an experiemental state of Video enhancement implementation
+ *
+ * \return bool Returns true if it is experimental yet
+ */
+bool SystemProperties::isVideoEnhancementExperimental()
+{
+    VideoEnhancement* videoEnhancement = &VideoEnhancement::getInstance();
+    return videoEnhancement->isExperimental();
+}

app/backend/systemproperties.h

@@ -32,6 +32,8 @@ class SystemProperties : public QObject
     Q_INVOKABLE QRect getNativeResolution(int displayIndex);
     Q_INVOKABLE QRect getSafeAreaResolution(int displayIndex);
     Q_INVOKABLE int getRefreshRate(int displayIndex);
+    Q_INVOKABLE bool isVideoEnhancementCapable();
+    Q_INVOKABLE bool isVideoEnhancementExperimental();
 
 signals:
     void unmappedGamepadsChanged();
@@ -59,4 +61,3 @@ class SystemProperties : public QObject
     bool supportsHdr;
     bool usesMaterial3Theme;
 };
-

app/cli/commandlineparser.cpp

@@ -59,13 +59,13 @@ class CommandLineParser : public QCommandLineParser
 
     void showMessage(QString message, MessageType type) const
     {
-    #if defined(Q_OS_WIN32)
+#if defined(Q_OS_WIN32)
         UINT flags = MB_OK | MB_TOPMOST | MB_SETFOREGROUND;
         flags |= (type == Info ? MB_ICONINFORMATION : MB_ICONERROR);
         QString title = "Moonlight";
         MessageBoxW(nullptr, reinterpret_cast<const wchar_t *>(message.utf16()),
                     reinterpret_cast<const wchar_t *>(title.utf16()), flags);
-    #endif
+#endif
         message = message.endsWith('\n') ? message : message + '\n';
         fputs(qPrintable(message), type == Info ? stdout : stderr);
     }
@@ -170,7 +170,7 @@ GlobalCommandLineParser::ParseResult GlobalCommandLineParser::parse(const QStrin
         "  pair            Pair a new host\n"
         "\n"
         "See 'moonlight <action> --help' for help of specific action."
-    );
+        );
     parser.addPositionalArgument("action", "Action to execute", "<action>");
     parser.parse(args);
     auto posArgs = parser.positionalArguments();
@@ -221,7 +221,7 @@ void QuitCommandLineParser::parse(const QStringList &args)
     parser.setApplicationDescription(
         "\n"
         "Quit the currently running app on the given host."
-    );
+        );
     parser.addPositionalArgument("quit", "quit running app");
     parser.addPositionalArgument("host", "Host computer name, UUID, or IP address", "<host>");
 
@@ -263,7 +263,7 @@ void PairCommandLineParser::parse(const QStringList &args)
     parser.setApplicationDescription(
         "\n"
         "Pair with the specified host."
-    );
+        );
     parser.addPositionalArgument("pair", "pair host");
     parser.addPositionalArgument("host", "Host computer name, UUID, or IP address", "<host>");
     parser.addValueOption("pin", "4 digit pairing PIN");
@@ -303,31 +303,31 @@ QString PairCommandLineParser::getPredefinedPin() const
 StreamCommandLineParser::StreamCommandLineParser()
 {
     m_WindowModeMap = {
-        {"fullscreen", StreamingPreferences::WM_FULLSCREEN},
-        {"windowed",   StreamingPreferences::WM_WINDOWED},
-        {"borderless", StreamingPreferences::WM_FULLSCREEN_DESKTOP},
-    };
+                       {"fullscreen", StreamingPreferences::WM_FULLSCREEN},
+                       {"windowed",   StreamingPreferences::WM_WINDOWED},
+                       {"borderless", StreamingPreferences::WM_FULLSCREEN_DESKTOP},
+                       };
     m_AudioConfigMap = {
-        {"stereo",       StreamingPreferences::AC_STEREO},
-        {"5.1-surround", StreamingPreferences::AC_51_SURROUND},
-        {"7.1-surround", StreamingPreferences::AC_71_SURROUND},
-    };
+                        {"stereo",       StreamingPreferences::AC_STEREO},
+                        {"5.1-surround", StreamingPreferences::AC_51_SURROUND},
+                        {"7.1-surround", StreamingPreferences::AC_71_SURROUND},
+                        };
     m_VideoCodecMap = {
-        {"auto",  StreamingPreferences::VCC_AUTO},
-        {"H.264", StreamingPreferences::VCC_FORCE_H264},
-        {"HEVC",  StreamingPreferences::VCC_FORCE_HEVC},
-        {"AV1", StreamingPreferences::VCC_FORCE_AV1},
-    };
+                       {"auto",  StreamingPreferences::VCC_AUTO},
+                       {"H.264", StreamingPreferences::VCC_FORCE_H264},
+                       {"HEVC",  StreamingPreferences::VCC_FORCE_HEVC},
+                       {"AV1", StreamingPreferences::VCC_FORCE_AV1},
+                       };
     m_VideoDecoderMap = {
-        {"auto",     StreamingPreferences::VDS_AUTO},
-        {"software", StreamingPreferences::VDS_FORCE_SOFTWARE},
-        {"hardware", StreamingPreferences::VDS_FORCE_HARDWARE},
-    };
+                         {"auto",     StreamingPreferences::VDS_AUTO},
+                         {"software", StreamingPreferences::VDS_FORCE_SOFTWARE},
+                         {"hardware", StreamingPreferences::VDS_FORCE_HARDWARE},
+                         };
     m_CaptureSysKeysModeMap = {
-        {"never",      StreamingPreferences::CSK_OFF},
-        {"fullscreen", StreamingPreferences::CSK_FULLSCREEN},
-        {"always",     StreamingPreferences::CSK_ALWAYS},
-    };
+                               {"never",      StreamingPreferences::CSK_OFF},
+                               {"fullscreen", StreamingPreferences::CSK_FULLSCREEN},
+                               {"always",     StreamingPreferences::CSK_ALWAYS},
+                               };
 }
 
 StreamCommandLineParser::~StreamCommandLineParser()
@@ -341,7 +341,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     parser.setApplicationDescription(
         "\n"
         "Starts directly streaming a given app."
-    );
+        );
     parser.addPositionalArgument("stream", "Start stream");
 
     // Add other arguments and options
@@ -367,6 +367,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     parser.addToggleOption("game-optimization", "game optimizations");
     parser.addToggleOption("audio-on-host", "audio on host PC");
     parser.addToggleOption("frame-pacing", "frame pacing");
+    parser.addToggleOption("video-enhancement", "Enhance video with AI");
     parser.addToggleOption("mute-on-focus-loss", "mute audio when Moonlight window loses focus");
     parser.addToggleOption("background-gamepad", "background gamepad input");
     parser.addToggleOption("reverse-scroll-direction", "inverted scroll direction");
@@ -474,6 +475,9 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     // Resolve --frame-pacing and --no-frame-pacing options
     preferences->framePacing = parser.getToggleOptionValue("frame-pacing", preferences->framePacing);
 
+    // Resolve --video-enhancement and --no-video-enhancement options
+    preferences->videoEnhancement = parser.getToggleOptionValue("video-enhancement", preferences->videoEnhancement);
+
     // Resolve --mute-on-focus-loss and --no-mute-on-focus-loss options
     preferences->muteOnFocusLoss = parser.getToggleOptionValue("mute-on-focus-loss", preferences->muteOnFocusLoss);
 
@@ -497,7 +501,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
 
     // Resolve --yuv444 and --no-yuv444 options
     preferences->enableYUV444 = parser.getToggleOptionValue("yuv444", preferences->enableYUV444);
-    
+
     // Resolve --capture-system-keys option
     if (parser.isSet("capture-system-keys")) {
         preferences->captureSysKeysMode = mapValue(m_CaptureSysKeysModeMap, parser.getChoiceOptionValue("capture-system-keys"));
@@ -555,7 +559,7 @@ void ListCommandLineParser::parse(const QStringList &args)
     parser.setApplicationDescription(
         "\n"
         "List the available apps on the given host."
-    );
+        );
     parser.addPositionalArgument("list", "list available apps");
     parser.addPositionalArgument("host", "Host computer name, UUID, or IP address", "<host>");
 

app/gui/SettingsView.qml

@@ -820,6 +820,44 @@ Flickable {
                     ToolTip.visible: hovered
                     ToolTip.text: qsTr("Frame pacing reduces micro-stutter by delaying frames that come in too early")
                 }
+
+                CheckBox {
+                    id: videoEnhancementCheck
+                    width: parent.width
+                    hoverEnabled: true
+                    text: qsTr("Video AI-Enhancement")
+                    font.pointSize:  12
+                    enabled: SystemProperties.isVideoEnhancementCapable()
+                    checked: {
+                        return SystemProperties.isVideoEnhancementCapable() && StreamingPreferences.videoEnhancement
+                    }
+                    property bool keepValue: checked;
+                    onCheckedChanged: {
+                        StreamingPreferences.videoEnhancement = checked
+                    }
+                    ToolTip.delay: 1000
+                    ToolTip.timeout: 5000
+                    ToolTip.visible: hovered
+                    ToolTip.text:
+                        qsTr("Enhance video quality by utilizing the GPU's AI-Enhancement capabilities.")
+                        + qsTr("\nThis feature effectively upscales, reduces compression artifacts and enhances the clarity of streamed content.")
+                        + qsTr("\nNote:")
+                        + qsTr("\n  - If available, ensure that appropriate settings (i.e. RTX Video enhancement) are enabled in your GPU driver configuration.")
+                        + qsTr("\n  - HDR rendering has divers issues depending on the GPU used, we are working on it but we advise to currently use Non-HDR.")
+                        + qsTr("\n  - Be advised that using this feature on laptops running on battery power may lead to significant battery drain.")
+
+                    Component.onCompleted: {
+                        if (!SystemProperties.isVideoEnhancementCapable()){
+                            // VSR or SDR->HDR feature could not be initialized by any GPU available
+                            text = qsTr("Video AI-Enhancement (Not supported by the GPU)")
+                            enabled = false;
+                            checked = false;
+                        } else if(SystemProperties.isVideoEnhancementExperimental()){
+                            // Indicate if the feature is available but not officially deployed by the Vendor
+                            text = qsTr("Video AI-Enhancement (Experimental)")
+                        }
+                    }
+                }
             }
         }
 
@@ -1176,7 +1214,7 @@ Flickable {
                         ListElement {
                             text: qsTr("Maximized")
                             val: StreamingPreferences.UI_MAXIMIZED
-                        }   
+                        }
                         ListElement {
                             text: qsTr("Fullscreen")
                             val: StreamingPreferences.UI_FULLSCREEN
@@ -1523,6 +1561,16 @@ Flickable {
                             StreamingPreferences.videoDecoderSelection = decoderListModel.get(currentIndex).val
                         }
                     }
+                    onCurrentIndexChanged: {
+                        if(decoderListModel.get(currentIndex).val === StreamingPreferences.VDS_FORCE_SOFTWARE){
+                            videoEnhancementCheck.enabled = false;
+                            videoEnhancementCheck.keepValue = videoEnhancementCheck.checked;
+                            videoEnhancementCheck.checked = false;
+                        } else {
+                            videoEnhancementCheck.enabled = true;
+                            videoEnhancementCheck.checked = videoEnhancementCheck.keepValue;
+                        }
+                    }
                 }
 
                 Label {