Allow virtio_gpu to auto-fallback to a safe rutabaga config

src/devices/src/virtio/gpu/virtio_gpu.rs

@@ -210,17 +210,14 @@ impl VirtioGpu {
         })
     }
 
-    #[allow(clippy::too_many_arguments)]
-    pub fn new(
+    pub fn create_rutabaga(
         mem: GuestMemoryMmap,
         queue_ctl: Arc<Mutex<VirtQueue>>,
         interrupt: InterruptTransport,
+        fence_state: Arc<Mutex<FenceState>>,
         virgl_flags: u32,
-        #[cfg(target_os = "macos")] map_sender: Sender<WorkerMessage>,
         export_table: Option<ExportTable>,
-        displays: Box<[DisplayInfo]>,
-        display_backend: DisplayBackend,
-    ) -> Self {
+    ) -> Option<Rutabaga> {
         let xdg_runtime_dir = match env::var("XDG_RUNTIME_DIR") {
             Ok(dir) => dir,
             Err(_) => "/run/user/1000".to_string(),
@@ -274,12 +271,62 @@ impl VirtioGpu {
             builder
         };
 
-        let fence_state = Arc::new(Mutex::new(Default::default()));
         let fence =
             Self::create_fence_handler(mem, queue_ctl.clone(), fence_state.clone(), interrupt);
-        let rutabaga = builder
-            .build(fence, None)
-            .expect("Rutabaga initialization failed!");
+        builder.clone().build(fence.clone(), None).ok()
+    }
+
+    pub fn create_fallback_rutabaga(
+        mem: GuestMemoryMmap,
+        queue_ctl: Arc<Mutex<VirtQueue>>,
+        interrupt: InterruptTransport,
+        fence_state: Arc<Mutex<FenceState>>,
+    ) -> Option<Rutabaga> {
+        const VIRGLRENDERER_NO_VIRGL: u32 = 1 << 7;
+        let builder = RutabagaBuilder::new(
+            rutabaga_gfx::RutabagaComponentType::VirglRenderer,
+            VIRGLRENDERER_NO_VIRGL,
+            0,
+        );
+
+        let fence =
+            Self::create_fence_handler(mem, queue_ctl.clone(), fence_state.clone(), interrupt);
+        builder.clone().build(fence.clone(), None).ok()
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        mem: GuestMemoryMmap,
+        queue_ctl: Arc<Mutex<VirtQueue>>,
+        interrupt: InterruptTransport,
+        virgl_flags: u32,
+        #[cfg(target_os = "macos")] map_sender: Sender<WorkerMessage>,
+        export_table: Option<ExportTable>,
+        displays: Box<[DisplayInfo]>,
+        display_backend: DisplayBackend,
+    ) -> Self {
+        let fence_state = Arc::new(Mutex::new(Default::default()));
+
+        let rutabaga = match Self::create_rutabaga(
+            mem.clone(),
+            queue_ctl.clone(),
+            interrupt.clone(),
+            fence_state.clone(),
+            virgl_flags,
+            export_table.clone(),
+        ) {
+            Some(rutabaga) => rutabaga,
+            None => {
+                error!("Failed to create virtio_gpu backend with the requested parameters");
+                Self::create_fallback_rutabaga(
+                    mem.clone(),
+                    queue_ctl.clone(),
+                    interrupt.clone(),
+                    fence_state.clone(),
+                )
+                .expect("Fallback rutabaga initialization failed")
+            }
+        };
 
         let display_backend = display_backend
             .create_instance()

src/rutabaga_gfx/src/virgl_renderer.rs

@@ -301,17 +301,6 @@ impl VirglRenderer {
             }
         }
 
-        // virglrenderer is a global state backed library that uses thread bound OpenGL contexts.
-        // Initialize it only once and use the non-send/non-sync Renderer struct to keep things tied
-        // to whichever thread called this function first.
-        static INIT_ONCE: AtomicBool = AtomicBool::new(false);
-        if INIT_ONCE
-            .compare_exchange(false, true, Ordering::Acquire, Ordering::Acquire)
-            .is_err()
-        {
-            return Err(RutabagaError::AlreadyInUse);
-        }
-
         unsafe { virgl_set_debug_callback(Some(debug_callback)) };
 
         // Cookie is intentionally never freed because virglrenderer never gets uninitialized.
@@ -335,6 +324,19 @@ impl VirglRenderer {
             )
         };
 
+        if ret == 0 {
+            // virglrenderer is a global state backed library that uses thread bound OpenGL contexts.
+            // Initialize it only once and use the non-send/non-sync Renderer struct to keep things tied
+            // to whichever thread called this function first.
+            static INIT_ONCE: AtomicBool = AtomicBool::new(false);
+            if INIT_ONCE
+                .compare_exchange(false, true, Ordering::Acquire, Ordering::Acquire)
+                .is_err()
+            {
+                return Err(RutabagaError::AlreadyInUse);
+            }
+        }
+
         ret_to_res(ret)?;
         Ok(Box::new(VirglRenderer {}))
     }