Skip to content

Commit a2b236f

Browse files
committed
wip
1 parent 893ba54 commit a2b236f

File tree

7 files changed

+652
-0
lines changed

7 files changed

+652
-0
lines changed

Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ members = [
1313
"flow-info",
1414
"hardware",
1515
"id",
16+
"init",
1617
"interface-manager",
1718
"mgmt",
1819
"nat",
@@ -45,6 +46,7 @@ flow-info = { path = "./flow-info", package = "dataplane-flow-info" }
4546
gateway_config = { git = "https://github.com/githedgehog/gateway-proto", tag = "v0.14.0", version = "0.14.0"}
4647
hardware = { path = "./hardware", package = "dataplane-hardware" }
4748
id = { path = "./id", package = "dataplane-id" }
49+
init = { path = "./init", package = "dataplane-init" }
4850
interface-manager = { path = "./interface-manager", package = "dataplane-interface-manager" }
4951
lpm = { path = "./lpm", package = "dataplane-lpm" }
5052
mgmt = { path = "./mgmt", package = "dataplane-mgmt" }

init/Cargo.toml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
[package]
2+
name = "dataplane-init"
3+
version = "0.1.0"
4+
edition = "2024"
5+
publish = false
6+
license = "Apache-2.0"
7+
8+
[dependencies]
9+
# internal
10+
hardware = { workspace = true, features = ["serde", "scan"] }
11+
id = { workspace = true }
12+
13+
# external
14+
nix = { workspace = true, features = ["mount", "fs"] }
15+
procfs = { workspace = true, features = [] }
16+
strum = { workspace = true, features = ["derive"] }
17+
strum_macros = { workspace = true, features = [] }
18+
thiserror = { workspace = true }
19+
tracing = { workspace = true, features = ["attributes"] }
20+
tracing-subscriber = { workspace = true, features = ["fmt"] }
21+
22+
[dev-dependencies]
23+
# internal
24+
25+
# external
26+
27+
[build-dependencies]
28+
# internal
29+
dpdk-sysroot-helper = { workspace = true }
30+
31+
# external

init/README.md

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# dataplane-init
2+
3+
This program is responsible for initializing the dataplane.
4+
5+
The primary steps of this program are to
6+
7+
1. drive the NIC into the configuration needed by DPDK to use the NIC.
8+
2. (todo) drop hazardous privileges,
9+
3. (todo) `exec` the dataplane process.
10+
11+
For most network cards, this configuration step involves unbinding the NIC from
12+
the kernel driver and re-binding it to the vfio-pci driver.
13+
14+
> [!NOTE]
15+
> Not all NICs should be bound to vfio-pci.
16+
> Some network cards use the so called bifurcated driver and must remain bound
17+
> to the kernel driver.
18+
> In particular, all network cards which use the mlx5 driver must remain bound
19+
> to the mlx5 kernel driver.
20+
21+
> [!WARNING]
22+
> This program is _not_ responsible for full life cycle management of the NIC.
23+
> In particular, it makes no attempt to rebind the NIC back to the kernel driver.
24+
> Thus, expect this program to make a network cards disappear from the eyes of
25+
> tooling like iproute2 and ethtool.
26+
27+
Only a very limited set of network cards are currently supported, although this set
28+
can easily be expanded over time.
29+
30+
## Error handling notes
31+
32+
As a short lived program which is only run once per gateway initialization,
33+
this program has significantly different error handling requirements from the
34+
other software in this workspace.
35+
36+
Basically, it will either succeed or fail, and if it fails it will likely require
37+
outside intervention to recover.
38+
There is little we can or should even try to do in the way of error handling beyond
39+
40+
## Privileges
41+
42+
This program is, by necessity, run with elevated privileges.
43+
As such, we need to take special caution when writing to files.
44+
45+
Because sysfs is basically a maze of symlinks, it is important to be careful when
46+
manipulating paths under sysfs.
47+
Mistakes can lead you to write data in highly unexpected places, with totally unknown
48+
consequences.
49+
Some care has been taken in the design of the types used here to discourage programmer
50+
errors which might lead to precipitous writes by a privileged process.

init/build.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright Open Network Fabric Authors
3+
4+
fn main() {
5+
let sysroot = dpdk_sysroot_helper::get_sysroot();
6+
println!("cargo:rustc-link-search=all={sysroot}/lib");
7+
println!("cargo:rustc-link-arg=--sysroot={sysroot}");
8+
}

init/src/main.rs

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright Open Network Fabric Authors
3+
4+
#![doc = include_str!("../README.md")]
5+
6+
pub mod nic;
7+
pub mod sysfs;
8+
9+
use std::path::PathBuf;
10+
11+
use sysfs::*;
12+
use tracing::{error, info};
13+
14+
use crate::nic::{BindToVfioPci, PciDriver, PciNic};
15+
16+
/// Errors which might occur during system dataplane system initialization
17+
#[derive(Debug, thiserror::Error)]
18+
pub enum InitErr {
19+
/// The path is not under a mounted sysfs and therefore does not qualify as a [`SysfsPath`].
20+
#[error("path {0:?} is not under sysfs")]
21+
PathNotUnderSysfs(PathBuf),
22+
/// Some [`std::io::Error`] error occurred
23+
#[error(transparent)]
24+
IoError(#[from] std::io::Error),
25+
/// Invalid UTF-8 in a path under sysfs is an absolutely wild error case I expect to
26+
/// never see.
27+
///
28+
/// The kernel just uses ascii byte strings for sysfs, so you should never see this
29+
/// error under healthy conditions.
30+
/// In the event of illegal UTF-8, something is likely deeply wrong with the kernel;
31+
/// likely memory corruption or some other security impacting issue.
32+
///
33+
/// As such, the [`InnitErr::InvalidUtf8`] branch deliberately does not include
34+
/// any information about the offending string name or any derivative, even for logging
35+
/// or error reporting.
36+
/// At best you will just end up mangling the log with unknown/unprintable bytes.
37+
/// At worst, injecting arbitrary bytes into a system log may be what an attacker needs
38+
/// for lateral compromise of some other system.
39+
///
40+
/// You should likely `panic!` if you reach this error case as there is no plausible
41+
/// recovery from this type of low level operating system malfunction.
42+
#[error("path under sysfs is not a valid UTF-8 string")]
43+
SysfsPathIsNotValidUtf8,
44+
#[error("missing needed driver {0}")]
45+
DriverMissing(PciDriver),
46+
}
47+
48+
/// Process setup helper function.
49+
///
50+
/// Fills out the [`SYSFS`] [`OnceLock`].
51+
///
52+
/// # Panics
53+
///
54+
/// - Panics if this process is unable to read the list of mounted filesystems.
55+
/// -
56+
///
57+
/// Panics if the [`SYSFS`] [`OnceLock`] cannot be initialized.
58+
fn setup() -> Result<(), InitErr> {
59+
tracing_subscriber::fmt()
60+
.with_ansi(false)
61+
.with_file(true)
62+
.with_level(true)
63+
.with_line_number(true)
64+
.init();
65+
let sysfs_mounts: Vec<_> = procfs::mounts()
66+
.unwrap() // acceptable panic: if we can't find /sys then this process will never work
67+
.into_iter()
68+
.filter(|mount| mount.fs_vfstype == "sysfs")
69+
.collect();
70+
let sysfs_path = if sysfs_mounts.is_empty() {
71+
panic!("sysfs is not mounted: unable to initialize dataplane");
72+
} else if sysfs_mounts.len() > 1 {
73+
const MSG_PREFIX: &str =
74+
"suspicious configuration found: sysfs is mounted at more than one location.";
75+
let message = std::format_args!("{MSG_PREFIX}. Filesystems found at {sysfs_mounts:#?}");
76+
error!("{message}");
77+
panic!("{message}");
78+
} else {
79+
sysfs_mounts[0].fs_file.clone()
80+
};
81+
let sysfs_root = SysfsPath::new(&sysfs_path)?;
82+
info!("found sysfs filesystem at {sysfs_root}");
83+
match SYSFS.set(sysfs_root) {
84+
Ok(()) => Ok(()),
85+
Err(_) => {
86+
unreachable!("called setup function twice in one process!");
87+
}
88+
}
89+
}
90+
91+
fn main() -> Result<(), InitErr> {
92+
setup()?;
93+
// TODO: proper argument parsing
94+
// -- hack add a real command line parser
95+
let mut args = std::env::args().skip(1);
96+
// -- end hack
97+
// TODO: fix unwraps in the next PR. These can't be properly addressed before the arg parser is done.
98+
let address = hardware::pci::address::PciAddress::try_from(args.next().unwrap()).unwrap();
99+
let mut device = PciNic::new(address)?;
100+
device.bind_to_vfio_pci()
101+
}

0 commit comments

Comments
 (0)