Auto merge of #608 - lennart/CLOUD-150, r=TroyKomodo,philipch07

scuffle-brawl[bot] · web-flow · commit 0d2cab507c08 · 2025-10-04T21:41:36.000Z
feat(ingest): ingest scaffolding This PR adds all the boilerplate for the ingest service. This doesn't implement anything. CLOUD-150 Requested-by: lennartkloock <39778085+lennartkloock@users.noreply.github.com> Reviewed-by: TroyKomodo <49777269+TroyKomodo@users.noreply.github.com> Reviewed-by: philipch07 <59272129+philipch07@users.noreply.github.com>
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -11,6 +11,7 @@ members = [
     "cloud/geo-ip",
     "cloud/id",
     "cloud/proto",
+    "cloud/video/ingest",
     "crates/aac",
     "crates/amf0",
     "crates/av1",
diff --git a/Justfile b/Justfile
@@ -43,6 +43,8 @@ run bin *args:
         bazel run //cloud/core:bin -- {{ args }}
     elif [ {{ bin }} == "email" ]; then
         bazel run //cloud/email:bin -- {{ args }}
+    elif [ {{ bin }} == "ingest" ]; then
+        bazel run //cloud/video/ingest:bin -- {{ args }}
     else
         echo "Unknown binary: {{ bin }}"
         exit 1
diff --git a/cargo_targets.bzl b/cargo_targets.bzl
@@ -8,6 +8,7 @@ _packages = [
     "//cloud/email",
     "//cloud/email/traits",
     "//cloud/geo-ip",
+    "//cloud/video/ingest",
     "//cloud/id",
     "//cloud/proto",
     "//crates/aac",
diff --git a/cloud/video/BUILD.bazel b/cloud/video/BUILD.bazel
diff --git a/cloud/video/ingest/BUILD.bazel b/cloud/video/ingest/BUILD.bazel
@@ -0,0 +1,35 @@
+load("//misc/utils/rust:manifest.bzl", "cargo_toml")
+load("//misc/utils/rust:package.bzl", "scuffle_package")
+
+cargo_toml()
+
+deps = [
+    "//crates/bootstrap",
+    "//crates/bootstrap-telemetry",
+    "//crates/context",
+    "//crates/settings",
+    "//crates/signal",
+    "//cloud/video/ingest/traits",
+]
+
+aliases = {
+    "//cloud/video/ingest/traits": "ingest_traits",
+}
+
+scuffle_package(
+    aliases = aliases,
+    crate_name = "scufflecloud-ingest",
+    deps = deps,
+)
+
+scuffle_package(
+    name = "bin",
+    srcs = glob([
+        "bin/standalone/**/*.rs",
+    ]),
+    aliases = aliases,
+    crate_name = "scufflecloud-ingest-standalone",
+    crate_type = "bin",
+    readme = False,
+    deps = deps + [":ingest"],
+)
diff --git a/cloud/video/ingest/Cargo.toml b/cloud/video/ingest/Cargo.toml
@@ -0,0 +1,37 @@
+[package]
+name = "scufflecloud-ingest"
+version = "0.1.0"
+authors = ["Scuffle <opensource@scuffle.cloud>"]
+edition = "2024"
+license = "AGPL-3.0"
+publish = false
+readme = "README.md"
+repository = "https://github.com/scufflecloud/scuffle"
+description = "Video ingest server for scuffle.cloud"
+
+[[bin]]
+name = "scufflecloud-ingest-standalone"
+path = "bin/standalone/main.rs"
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage_nightly)'] }
+
+[dependencies]
+anyhow = "1.0.98"
+ingest-traits = { path = "./traits", package = "scufflecloud-ingest-traits" }
+scuffle-bootstrap = { path = "../../../crates/bootstrap" }
+scuffle-bootstrap-telemetry = { features = ["opentelemetry-logs", "opentelemetry-traces"], path = "../../../crates/bootstrap-telemetry" }
+scuffle-context = { path = "../../../crates/context" }
+scuffle-settings = { features = ["all-formats", "bootstrap"], path = "../../../crates/settings" }
+scuffle-signal = { features = ["bootstrap"], path = "../../../crates/signal" }
+serde = "1.0.219"
+serde_derive = "1.0.219"
+smart-default = "0.7.1"
+tracing = "0.1.41"
+tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }
+
+[package.metadata.sync-readme.badges]
+docs-rs = false
+crates-io = false
+license = true
+codecov = true
diff --git a/cloud/video/ingest/LICENSE.AGPL-3.0 b/cloud/video/ingest/LICENSE.AGPL-3.0
@@ -0,0 +1 @@
+../../../LICENSE.AGPL-3.0
diff --git a/cloud/video/ingest/README.md b/cloud/video/ingest/README.md
@@ -0,0 +1,21 @@
+<!-- dprint-ignore-file -->
+<!-- sync-readme title [[ -->
+# scufflecloud-ingest
+<!-- sync-readme ]] -->
+
+<!-- sync-readme badge [[ -->
+![License: AGPL-3.0](https://img.shields.io/badge/license-AGPL--3.0-purple.svg?style=flat-square)
+[![Codecov](https://img.shields.io/codecov/c/github/scufflecloud/scuffle.svg?label=codecov&logo=codecov&style=flat-square)](https://app.codecov.io/gh/scufflecloud/scuffle)
+<!-- sync-readme ]] -->
+
+---
+
+<!-- sync-readme rustdoc [[ -->
+Video ingest server for <https://scuffle.cloud/>.
+
+### License
+
+This project is licensed under the [AGPL-3.0](./LICENSE.AGPL-3.0).
+
+`SPDX-License-Identifier: AGPL-3.0`
+<!-- sync-readme ]] -->
diff --git a/cloud/video/ingest/bin/standalone/config.rs b/cloud/video/ingest/bin/standalone/config.rs
@@ -0,0 +1,19 @@
+use std::net::SocketAddr;
+
+#[derive(serde_derive::Deserialize, smart_default::SmartDefault, Debug, Clone)]
+#[serde(default)]
+pub(crate) struct Config {
+    #[default(env!("CARGO_PKG_NAME").to_string())]
+    pub service_name: String,
+    #[default = "info"]
+    pub level: String,
+    pub telemetry: Option<TelemetryConfig>,
+}
+
+scuffle_settings::bootstrap!(Config);
+
+#[derive(serde_derive::Deserialize, smart_default::SmartDefault, Debug, Clone)]
+pub(crate) struct TelemetryConfig {
+    #[default("[::1]:4317".parse().unwrap())]
+    pub bind: SocketAddr,
+}
diff --git a/cloud/video/ingest/bin/standalone/main.rs b/cloud/video/ingest/bin/standalone/main.rs
@@ -0,0 +1,73 @@
+#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+// #![deny(missing_docs)]
+#![deny(unsafe_code)]
+#![deny(unreachable_pub)]
+#![deny(clippy::mod_module_files)]
+
+use std::sync::Arc;
+
+use scuffle_bootstrap_telemetry::opentelemetry;
+use scuffle_bootstrap_telemetry::opentelemetry_sdk::logs::SdkLoggerProvider;
+use scuffle_bootstrap_telemetry::opentelemetry_sdk::trace::SdkTracerProvider;
+use tracing_subscriber::Layer;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+
+mod config;
+
+struct Global {
+    config: config::Config,
+    open_telemetry: opentelemetry::OpenTelemetry,
+}
+
+impl ingest_traits::Global for Global {}
+
+impl scuffle_signal::SignalConfig for Global {}
+
+impl scuffle_bootstrap_telemetry::TelemetryConfig for Global {
+    fn enabled(&self) -> bool {
+        self.config.telemetry.is_some()
+    }
+
+    fn bind_address(&self) -> Option<std::net::SocketAddr> {
+        self.config.telemetry.as_ref().map(|telemetry| telemetry.bind)
+    }
+
+    fn http_server_name(&self) -> &str {
+        "scufflecloud-ingest-telemetry"
+    }
+
+    fn opentelemetry(&self) -> Option<&opentelemetry::OpenTelemetry> {
+        Some(&self.open_telemetry)
+    }
+}
+
+impl scuffle_bootstrap::Global for Global {
+    type Config = config::Config;
+
+    async fn init(config: Self::Config) -> anyhow::Result<Arc<Self>> {
+        tracing_subscriber::registry()
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_filter(tracing_subscriber::EnvFilter::from_default_env().add_directive(config.level.parse()?)),
+            )
+            .init();
+
+        let tracer = SdkTracerProvider::default();
+        opentelemetry::global::set_tracer_provider(tracer.clone());
+
+        let logger = SdkLoggerProvider::builder().build();
+
+        let open_telemetry = opentelemetry::OpenTelemetry::new().with_traces(tracer).with_logs(logger);
+
+        Ok(Arc::new(Self { config, open_telemetry }))
+    }
+}
+
+scuffle_bootstrap::main! {
+    Global {
+        scuffle_signal::SignalSvc,
+        scufflecloud_ingest::services::IngestSvc::<Global>::default(),
+    }
+}
diff --git a/cloud/video/ingest/src/lib.rs b/cloud/video/ingest/src/lib.rs
@@ -0,0 +1,17 @@
+//! Video ingest server for <https://scuffle.cloud/>.
+//!
+//! ## License
+//!
+//! This project is licensed under the [AGPL-3.0](./LICENSE.AGPL-3.0).
+//!
+//! `SPDX-License-Identifier: AGPL-3.0`
+#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+// #![deny(missing_docs)]
+#![deny(unsafe_code)]
+#![deny(unreachable_pub)]
+#![deny(clippy::mod_module_files)]
+// tonic::Status emits this warning
+#![allow(clippy::result_large_err)]
+
+pub mod services;
diff --git a/cloud/video/ingest/src/services.rs b/cloud/video/ingest/src/services.rs
@@ -0,0 +1,20 @@
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub struct IngestSvc<G> {
+    _phantom: std::marker::PhantomData<G>,
+}
+
+impl<G> Default for IngestSvc<G> {
+    fn default() -> Self {
+        Self {
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<G: ingest_traits::Global> scuffle_bootstrap::Service<G> for IngestSvc<G> {
+    async fn run(self, _global: Arc<G>, _ctx: scuffle_context::Context) -> anyhow::Result<()> {
+        Ok(())
+    }
+}
diff --git a/cloud/video/ingest/traits/BUILD.bazel b/cloud/video/ingest/traits/BUILD.bazel
@@ -0,0 +1,8 @@
+load("//misc/utils/rust:manifest.bzl", "cargo_toml")
+load("//misc/utils/rust:package.bzl", "scuffle_package")
+
+cargo_toml()
+
+scuffle_package(
+    crate_name = "scufflecloud-ingest-traits",
+)
diff --git a/cloud/video/ingest/traits/Cargo.toml b/cloud/video/ingest/traits/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "scufflecloud-ingest-traits"
+version = "0.1.0"
+authors = ["Scuffle <opensource@scuffle.cloud>"]
+edition = "2024"
+license = "AGPL-3.0"
+publish = false
+readme = "README.md"
+repository = "https://github.com/scufflecloud/scuffle"
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage_nightly)'] }
+
+[dependencies]
+
+[package.metadata.sync-readme.badges]
+docs-rs = false
+crates-io = false
+license = true
+codecov = true
diff --git a/cloud/video/ingest/traits/README.md b/cloud/video/ingest/traits/README.md
@@ -0,0 +1,13 @@
+<!-- dprint-ignore-file -->
+<!-- sync-readme title [[ -->
+# scufflecloud-ingest-traits
+<!-- sync-readme ]] -->
+
+<!-- sync-readme badge [[ -->
+![License: AGPL-3.0](https://img.shields.io/badge/license-AGPL--3.0-purple.svg?style=flat-square)
+[![Codecov](https://img.shields.io/codecov/c/github/scufflecloud/scuffle.svg?label=codecov&logo=codecov&style=flat-square)](https://app.codecov.io/gh/scufflecloud/scuffle)
+<!-- sync-readme ]] -->
+
+---
+
+<!-- sync-readme rustdoc -->
diff --git a/cloud/video/ingest/traits/src/lib.rs b/cloud/video/ingest/traits/src/lib.rs
@@ -0,0 +1,8 @@
+#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+// #![deny(missing_docs)]
+#![deny(unsafe_code)]
+#![deny(unreachable_pub)]
+#![deny(clippy::mod_module_files)]
+
+pub trait Global: Send + Sync + 'static {}
diff --git a/vendor/cargo/defs.bzl b/vendor/cargo/defs.bzl
