Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions blobby/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ categories = ["no-std"]
edition = "2024"
rust-version = "1.85"
readme = "README.md"

[features]
alloc = []
68 changes: 43 additions & 25 deletions blobby/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,47 @@ Iterators over a simple binary blob storage.

## Examples
```
let buf = b"\x02\x05hello\x06world!\x01\x02 \x00\x03\x06:::\x03\x01\x00";
let mut v = blobby::BlobIterator::new(buf).unwrap();
assert_eq!(v.next(), Some(Ok(&b"hello"[..])));
assert_eq!(v.next(), Some(Ok(&b" "[..])));
assert_eq!(v.next(), Some(Ok(&b""[..])));
assert_eq!(v.next(), Some(Ok(&b"world!"[..])));
assert_eq!(v.next(), Some(Ok(&b":::"[..])));
assert_eq!(v.next(), Some(Ok(&b"world!"[..])));
assert_eq!(v.next(), Some(Ok(&b"hello"[..])));
assert_eq!(v.next(), Some(Ok(&b""[..])));
assert_eq!(v.next(), None);

let mut v = blobby::Blob2Iterator::new(buf).unwrap();
assert_eq!(v.next(), Some(Ok([&b"hello"[..], b" "])));
assert_eq!(v.next(), Some(Ok([&b""[..], b"world!"])));
assert_eq!(v.next(), Some(Ok([&b":::"[..], b"world!"])));
assert_eq!(v.next(), Some(Ok([&b"hello"[..], b""])));
assert_eq!(v.next(), None);

let mut v = blobby::Blob4Iterator::new(buf).unwrap();
assert_eq!(v.next(), Some(Ok([&b"hello"[..], b" ", b"", b"world!"])));
assert_eq!(v.next(), Some(Ok([&b":::"[..], b"world!", b"hello", b""])));
assert_eq!(v.next(), None);
// We recommend to save blobby data into separate files and
// use the `include_bytes!` macro
static BLOBBY_DATA: &[u8] = b"\x02\x05hello\x06world!\x01\x02 \x00\x03\x06:::\x03\x01\x00";

static SLICE: &[&[u8]] = blobby::parse_into_slice!(BLOBBY_DATA);

assert_eq!(SLICE[0], b"hello".as_slice());
assert_eq!(SLICE[1], b" ".as_slice());
assert_eq!(SLICE[2], b"".as_slice());
assert_eq!(SLICE[3], b"world!".as_slice());
assert_eq!(SLICE[4], b":::".as_slice());
assert_eq!(SLICE[5], b"world!".as_slice());
assert_eq!(SLICE[6], b"hello".as_slice());
assert_eq!(SLICE[7], b"".as_slice());
assert_eq!(SLICE.len(), 8);

blobby::parse_into_structs!(
BLOBBY_DATA;
static ITEMS;
struct Foo { a, b, c, d }
);

assert_eq!(
ITEMS[0],
Foo {
a: b"hello",
b: b" ",
c: b"",
d: b"world!",
},
);
assert_eq!(
ITEMS[1],
Foo {
a: b":::",
b: b"world!",
c: b"hello",
d: b"",
},
);
assert_eq!(ITEMS.len(), 2);
```

## Encoding and decoding
Expand Down Expand Up @@ -76,14 +94,14 @@ with `\n`).

This file can be converted to the Blobby format by running the following command:
```sh
cargo run --releae --bin encode -- /path/to/input.txt /path/to/output.blb
cargo run --release --features alloc --bin encode -- /path/to/input.txt /path/to/output.blb
```

This will create a file which can be read using `blobby::Blob2Iterator`.

To see contents of an existing Blobby file you can use the following command:
```sh
cargo run --releae --bin decode -- /path/to/input.blb /path/to/output.txt
cargo run --release --features alloc --bin decode -- /path/to/input.blb /path/to/output.txt
```
The output file will contain a sequence of hex-encoded byte strings stored
in the input file.
Expand Down
56 changes: 28 additions & 28 deletions blobby/src/bin/decode.rs
Original file line number Diff line number Diff line change
@@ -1,41 +1,41 @@
//! Encoding utility
use blobby::BlobIterator;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, error::Error, fs::File};

fn encode_hex(data: &[u8]) -> String {
let mut res = String::with_capacity(2 * data.len());
for &byte in data {
res.push_str(&format!("{byte:02X}"));
}
res
use std::error::Error;

#[cfg(not(feature = "alloc"))]
fn main() -> Result<(), Box<dyn Error>> {
Err("The decode binary should be compiled with enabled `alloc` feature!".into())
}

fn decode<R: BufRead, W: Write>(mut reader: R, mut writer: W) -> io::Result<usize> {
let mut data = Vec::new();
reader.read_to_end(&mut data)?;
let res = BlobIterator::new(&data)
.map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid blobby data: {e:?}"),
)
})?
.collect::<Vec<_>>();
for blob in res.iter() {
let blob = blob.map_err(|e| {
#[cfg(feature = "alloc")]
fn main() -> Result<(), Box<dyn Error>> {
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, fs::File};

fn encode_hex(data: &[u8]) -> String {
let mut res = String::with_capacity(2 * data.len());
for &byte in data {
res.push_str(&format!("{byte:02X}"));
}
res
}

fn decode<R: BufRead, W: Write>(mut reader: R, mut writer: W) -> io::Result<usize> {
let mut data = Vec::new();
reader.read_to_end(&mut data)?;
let res = blobby::parse_into_vec(&data).map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid blobby data: {e:?}"),
)
})?;
writer.write_all(encode_hex(blob).as_bytes())?;
writer.write_all(b"\n")?;
let len = res.len();
for blob in res {
writer.write_all(encode_hex(blob).as_bytes())?;
writer.write_all(b"\n")?;
}
Ok(len)
}
Ok(res.len())
}

fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().skip(1).collect();

if args.is_empty() {
Expand Down
84 changes: 46 additions & 38 deletions blobby/src/bin/encode.rs
Original file line number Diff line number Diff line change
@@ -1,49 +1,57 @@
//! Encoding utility
use blobby::encode_blobs;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, error::Error, fs::File};
use std::error::Error;

fn decode_hex_char(b: u8) -> io::Result<u8> {
let res = match b {
b'0'..=b'9' => b - b'0',
b'a'..=b'f' => b - b'a' + 10,
b'A'..=b'F' => b - b'A' + 10,
_ => {
let msg = "Invalid hex string: invalid byte {b}";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
};
Ok(res)
#[cfg(not(feature = "alloc"))]
fn main() -> Result<(), Box<dyn Error>> {
Err("The encode binary should be compiled with enabled `alloc` feature!".into())
}

fn decode_hex(data: &str) -> io::Result<Vec<u8>> {
if data.len() % 2 != 0 {
let msg = "Invalid hex string: length is not even";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
#[cfg(feature = "alloc")]
fn main() -> Result<(), Box<dyn Error>> {
use blobby::encode_blobs;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, fs::File};

fn decode_hex_char(b: u8) -> io::Result<u8> {
let res = match b {
b'0'..=b'9' => b - b'0',
b'a'..=b'f' => b - b'a' + 10,
b'A'..=b'F' => b - b'A' + 10,
_ => {
let msg = "Invalid hex string: invalid byte {b}";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
};
Ok(res)
}
data.as_bytes()
.chunks_exact(2)
.map(|chunk| {
let a = decode_hex_char(chunk[0])?;
let b = decode_hex_char(chunk[1])?;
Ok((a << 4) | b)
})
.collect()
}

fn encode(reader: impl BufRead, mut writer: impl Write) -> io::Result<usize> {
let mut blobs = Vec::new();
for line in reader.lines() {
let blob = decode_hex(&line?)?;
blobs.push(blob);
fn decode_hex(data: &str) -> io::Result<Vec<u8>> {
if data.len() % 2 != 0 {
let msg = "Invalid hex string: length is not even";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
data.as_bytes()
.chunks_exact(2)
.map(|chunk| {
let a = decode_hex_char(chunk[0])?;
let b = decode_hex_char(chunk[1])?;
Ok((a << 4) | b)
})
.collect()
}

fn encode(reader: impl BufRead, mut writer: impl Write) -> io::Result<usize> {
let mut blobs = Vec::new();
for line in reader.lines() {
let blob = decode_hex(&line?)?;
blobs.push(blob);
}
let (data, idx_len) = encode_blobs(&blobs);
println!("Index len: {idx_len:?}");
writer.write_all(&data)?;
Ok(blobs.len())
}
let (data, idx_len) = encode_blobs(&blobs);
println!("Index len: {idx_len:?}");
writer.write_all(&data)?;
Ok(blobs.len())
}

fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().skip(1).collect();

if args.is_empty() {
Expand Down
Loading