+TLV blob read/write fns

Cargo.lock

@@ -1,5 +1,11 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "adler32"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
+
 [[package]]
 name = "ansi_term"
 version = "0.11.0"
@@ -44,6 +50,22 @@ version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742"
 
+[[package]]
+name = "byteorder"
+version = "1.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+
+[[package]]
+name = "bytes"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
+dependencies = [
+ "byteorder",
+ "iovec",
+]
+
 [[package]]
 name = "bytes"
 version = "0.5.4"
@@ -77,6 +99,29 @@ dependencies = [
  "vec_map",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "flate2"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "futures 0.1.29",
+ "libc",
+ "miniz_oxide",
+ "tokio-io",
+]
+
 [[package]]
 name = "fuchsia-zircon"
 version = "0.3.3"
@@ -93,6 +138,27 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 
+[[package]]
+name = "futures"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef"
+
+[[package]]
+name = "futures"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.4"
@@ -100,6 +166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8"
 dependencies = [
  "futures-core",
+ "futures-sink",
 ]
 
 [[package]]
@@ -108,6 +175,23 @@ version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a"
 
+[[package]]
+name = "futures-executor"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6"
+
 [[package]]
 name = "futures-macro"
 version = "0.3.4"
@@ -120,6 +204,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "futures-sink"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6"
+
 [[package]]
 name = "futures-task"
 version = "0.3.4"
@@ -132,9 +222,13 @@ version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5"
 dependencies = [
+ "futures-channel",
  "futures-core",
+ "futures-io",
  "futures-macro",
+ "futures-sink",
  "futures-task",
+ "memchr",
  "pin-utils",
  "proc-macro-hack",
  "proc-macro-nested",
@@ -214,6 +308,15 @@ version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa679ff6578b1cddee93d7e82e263b94a575e0bfced07284eb0c037c1d2416a5"
+dependencies = [
+ "adler32",
+]
+
 [[package]]
 name = "mio"
 version = "0.6.21"
@@ -503,7 +606,7 @@ version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fa5e81d6bc4e67fe889d5783bd2a128ab2e0cfa487e0be16b6a8d177b101616"
 dependencies = [
- "bytes",
+ "bytes 0.5.4",
  "iovec",
  "lazy_static",
  "libc",
@@ -514,6 +617,17 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "tokio-io"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
+dependencies = [
+ "bytes 0.4.12",
+ "futures 0.1.29",
+ "log",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.13.0"
@@ -544,6 +658,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
 
+[[package]]
+name = "unsigned-varint"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b7ffb36714206d2f5f05d61a2bc350415c642f2c54433f0ebf829afbe41d570"
+dependencies = [
+ "futures 0.3.4",
+]
+
 [[package]]
 name = "untrusted"
 version = "0.7.0"
@@ -727,10 +850,13 @@ dependencies = [
 name = "zsittle"
 version = "0.0.0"
 dependencies = [
+ "flate2",
+ "futures-io",
  "futures-util",
  "serde",
  "tokio",
  "tokio-rustls",
+ "unsigned-varint",
 ]
 
 [[package]]

zsittle/Cargo.toml

@@ -5,7 +5,10 @@ authors = ["Erik Zscheile <zseri.devel@ytrizja.de>"]
 edition = "2018"
 
 [dependencies]
-futures-util = "0.3.4"
+flate2 = { version = "1.0", features = ["tokio"] }
+futures-io = "0.3"
+futures-util = "0.3"
 serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "0.2", features = ["net", "io-std", "io-util", "rt-threaded"] }
 tokio-rustls = "0.13"
+unsigned-varint = { version = "0.3", features = ["futures"] }

zsittle/src/lib.rs

@@ -1 +1,64 @@
 pub use {futures_util, serde, tokio, tokio_rustls};
+
+use {
+    flate2::bufread as flate2r,
+    std::{
+        convert::TryInto,
+        io::{self, Read},
+        marker::Unpin,
+    },
+    tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
+};
+
+/// typ often corresponds to the used format version
+pub type BlobType = u32;
+
+pub struct Blob {
+    pub typ: BlobType,
+    pub data: Vec<u8>,
+}
+
+async fn write_u32<W>(writer: &mut W, value: u32) -> io::Result<()>
+where
+    W: AsyncWrite + Unpin,
+{
+    use unsigned_varint::encode as uve;
+    let mut buf = uve::u32_buffer();
+    let slc = uve::u32(value, &mut buf);
+    writer.write_all(slc).await?;
+    Ok(())
+}
+
+/// NOTE: we only support blobs with a size up to 2^31 bytes (2 Gibibytes).
+pub async fn read_blob<R>(r: &mut R) -> Result<Blob, unsigned_varint::io::ReadError>
+where
+    R: AsyncRead + Unpin + futures_io::AsyncRead,
+{
+    use unsigned_varint::aio::read_u32;
+
+    let typ = read_u32(&mut *r).await?;
+    let len = read_u32(&mut *r).await?.try_into().unwrap();
+
+    let mut datac = Vec::with_capacity(len);
+    datac.resize(len, 0);
+    r.read_exact(&mut datac[..]).await?;
+
+    let mut z = flate2r::DeflateDecoder::new(&datac[..]);
+    let mut data = Vec::new();
+    z.read_to_end(&mut data)?;
+    Ok(Blob { typ, data })
+}
+
+pub async fn write_blob<W>(w: &mut W, blob: Blob) -> io::Result<()>
+where
+    W: AsyncWrite + Unpin,
+{
+    let mut z = flate2r::DeflateEncoder::new(&blob.data[..], flate2::Compression::fast());
+    let mut datac = Vec::with_capacity(blob.data.len());
+    z.read_to_end(&mut datac)?;
+
+    write_u32(w, blob.typ).await?;
+    write_u32(w, datac.len().try_into().unwrap()).await?;
+    w.write_all(&datac[..]).await?;
+    Ok(())
+}