vo_gpu_next: separate pl_bit_encoding from pl_plane_data handling

So we can use it on hwdec frames too.

Author: haasn

video/out/vo_gpu_next.c

@@ -425,6 +425,30 @@ struct frame_priv {
     struct ra_hwdec *hwdec;
 };
 
+static struct pl_bit_encoding plane_bits_from_imgfmt(enum mp_imgfmt imgfmt)
+{
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(imgfmt);
+    struct pl_bit_encoding bits = {0};
+
+    const int num_comps = mp_imgfmt_desc_get_num_comps(&desc);
+    for (int c = 0; c < MPMIN(num_comps, 3); c++) {
+        struct pl_bit_encoding cbits = {
+            .sample_depth = desc.comps[c].size,
+            .color_depth  = desc.comps[c].size - abs(desc.comps[c].pad),
+            .bit_shift    = MPMAX(desc.comps[c].pad, 0),
+        };
+
+        if (bits.sample_depth && !pl_bit_encoding_equal(&bits, &cbits)) {
+            // Bit encoding differs between components, cannot handle this
+            return (struct pl_bit_encoding) {0};
+        }
+
+        bits = cbits;
+    }
+
+    return bits;
+}
+
 static int plane_data_from_imgfmt(struct pl_plane_data out_data[4],
                                   struct pl_bit_encoding *out_bits,
                                   enum mp_imgfmt imgfmt, bool use_uint)
@@ -445,7 +469,6 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4],
     if ((desc.flags & MP_IMGFLAG_TYPE_FLOAT) && (desc.flags & MP_IMGFLAG_YUV))
         return 0; // Floating-point YUV (currently) unsupported
 
-    bool has_bits = false;
     bool any_padded = false;
 
     for (int p = 0; p < desc.num_planes; p++) {
@@ -482,28 +505,6 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4],
             data->component_pad[c] = sorted[c].offset - total_bits;
             total_bits += data->component_pad[c] + data->component_size[c];
             any_padded |= sorted[c].pad;
-
-            // Ignore bit encoding of alpha channel
-            if (!out_bits || data->component_map[c] == PL_CHANNEL_A)
-                continue;
-
-            struct pl_bit_encoding bits = {
-                .sample_depth = data->component_size[c],
-                .color_depth = sorted[c].size - abs(sorted[c].pad),
-                .bit_shift = MPMAX(sorted[c].pad, 0),
-            };
-
-            if (!has_bits) {
-                *out_bits = bits;
-                has_bits = true;
-            } else {
-                if (!pl_bit_encoding_equal(out_bits, &bits)) {
-                    // Bit encoding differs between components/planes,
-                    // cannot handle this
-                    *out_bits = (struct pl_bit_encoding) {0};
-                    out_bits = NULL;
-                }
-            }
         }
 
         data->pixel_stride = desc.bpp[p] / 8;
@@ -512,8 +513,11 @@ static int plane_data_from_imgfmt(struct pl_plane_data out_data[4],
                             : (use_uint ? PL_FMT_UINT : PL_FMT_UNORM);
     }
 
-    if (any_padded && !out_bits)
+    if (out_bits) {
+        plane_bits_from_imgfmt(imgfmt);
+    } else if (any_padded) {
         return 0; // can't handle padded components without `pl_bit_encoding`
+    }
 
     return desc.num_planes;
 }