From ec3486bc6e3fefb13269a6bc51d960fdd3b81c2d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sun, 6 Apr 2025 15:13:53 -0500 Subject: [PATCH 01/32] update proptest to fix 'impl non-local' warning --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a5402bf..61f3db9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,8 +47,8 @@ random-access-disk = { version = "3", default-features = false } [dev-dependencies] anyhow = "1.0.70" -proptest = "1.1.0" -proptest-derive = "0.2.0" +proptest = "1.6.0" +proptest-derive = "0.5.1" data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" From a2cf32c5cb137786651e1f1ce85869629e17d0fb Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 8 Apr 2025 11:39:53 -0500 Subject: [PATCH 02/32] add CompactEncodable derives --- src/encoding.rs | 322 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) diff --git a/src/encoding.rs b/src/encoding.rs index ed049a6..a327160 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,4 +1,8 @@ //! Hypercore-specific compact encodings +use compact_encoding::encodable::{ + bytes_fixed_from_vec, encode_bytes_fixed, write_slice, VecEncodable, +}; +use compact_encoding::types::{take_array, usize_encoded_size, CompactEncodable}; pub use compact_encoding::{CompactEncoding, EncodingError, EncodingErrorKind, State}; use std::convert::TryInto; use std::ops::{Deref, DerefMut}; @@ -368,3 +372,321 @@ impl CompactEncoding for State { }) } } + +impl CompactEncodable for Node { + fn encoded_size(&self) -> Result { + Ok(self.index.encoded_size()? + self.length.encoded_size()? + 32) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.index.encoded_bytes(buffer)?; + let rest = self.length.encoded_bytes(rest)?; + bytes_fixed_from_vec::<32>(&self.hash)?.encoded_bytes(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (index, rest) = u64::decode(buffer)?; + let (length, rest) = u64::decode(rest)?; + let (hash, rest) = <[u8; 32]>::decode(rest)?; + Ok((Node::new(index, hash.to_vec(), length), rest)) + } +} + +impl VecEncodable for Node { + fn vec_encoded_size(vec: &[Self]) -> Result + where + Self: Sized, + { + let mut out = usize_encoded_size(vec.len()); + for x in vec { + out += x.encoded_size()?; + } + Ok(out) + } +} +impl CompactEncodable for RequestBlock { + fn encoded_size(&self) -> Result { + Ok(self.index.encoded_size()? + self.nodes.encoded_size()?) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.index.encoded_bytes(buffer)?; + let rest = self.nodes.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (index, rest) = u64::decode(buffer)?; + let (nodes, rest) = u64::decode(rest)?; + Ok((RequestBlock { index, nodes }, rest)) + } +} + +impl CompactEncodable for RequestSeek { + fn encoded_size(&self) -> Result { + self.bytes.encoded_size() + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + self.bytes.encoded_bytes(buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (bytes, rest) = u64::decode(buffer)?; + Ok((RequestSeek { bytes }, rest)) + } +} + +impl CompactEncodable for RequestUpgrade { + fn encoded_size(&self) -> Result { + Ok(self.start.encoded_size()? + self.length.encoded_size()?) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.start.encoded_bytes(buffer)?; + let rest = self.length.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (start, rest) = u64::decode(buffer)?; + let (length, rest) = u64::decode(rest)?; + Ok((RequestUpgrade { start, length }, rest)) + } +} + +impl CompactEncodable for DataBlock { + fn encoded_size(&self) -> Result { + Ok(self.index.encoded_size()? + self.value.encoded_size()? + self.nodes.encoded_size()?) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.index.encoded_bytes(buffer)?; + let rest = self.value.encoded_bytes(rest)?; + let rest = self.nodes.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (index, rest) = u64::decode(buffer)?; + let (value, rest) = Vec::::decode(rest)?; + let (nodes, rest) = Vec::::decode(rest)?; + Ok(( + DataBlock { + index, + value, + nodes, + }, + rest, + )) + } +} + +impl CompactEncodable for DataHash { + fn encoded_size(&self) -> Result { + Ok(self.index.encoded_size()? + self.nodes.encoded_size()?) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.index.encoded_bytes(buffer)?; + let rest = self.nodes.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (index, rest) = u64::decode(buffer)?; + let (nodes, rest) = Vec::::decode(rest)?; + Ok((DataHash { index, nodes }, rest)) + } +} + +impl CompactEncodable for DataSeek { + fn encoded_size(&self) -> Result { + Ok(self.bytes.encoded_size()? + self.nodes.encoded_size()?) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.bytes.encoded_bytes(buffer)?; + let rest = self.nodes.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (bytes, rest) = u64::decode(buffer)?; + let (nodes, rest) = Vec::::decode(rest)?; + Ok((DataSeek { bytes, nodes }, rest)) + } +} + +macro_rules! sum_encoded_size { + // Base case: single field + ($self:ident, $field:ident) => { + $self.$field.encoded_size()? + }; + // Recursive case: first field + rest + ($self: ident, $first:ident, $($rest:ident),+) => { + $self.$first.encoded_size()? + sum_encoded_size!($self, $($rest),+) + }; +} + +impl CompactEncodable for DataUpgrade { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!( + self, + start, + length, + nodes, + additional_nodes, + signature + )) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = self.start.encoded_bytes(buffer)?; + let rest = self.length.encoded_bytes(rest)?; + let rest = self.nodes.encoded_bytes(rest)?; + let rest = self.additional_nodes.encoded_bytes(rest)?; + let rest = self.signature.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (start, rest) = u64::decode(buffer)?; + let (length, rest) = u64::decode(rest)?; + let (nodes, rest) = Vec::::decode(rest)?; + let (additional_nodes, rest) = Vec::::decode(rest)?; + let (signature, rest) = <[u8; 32]>::decode(rest)?; + Ok(( + DataUpgrade { + start, + length, + nodes, + additional_nodes, + signature: signature.to_vec(), + }, + rest, + )) + } +} + +impl CompactEncodable for ManifestSigner { + fn encoded_size(&self) -> Result { + Ok( + 1 /* Signature */ + 32 /* namespace */ + 32, /* public_key */ + ) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = if &self.signature == "ed25519" { + write_slice(&[0], buffer)? + } else { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown signature type: {}", &self.signature), + )); + }; + let rest = encode_bytes_fixed(&self.namespace, rest)?; + encode_bytes_fixed(&self.public_key, rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([signature_id], rest) = take_array::<1>(buffer)?; + let signature: String = if signature_id != 0 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown signature id: {signature_id}"), + )); + } else { + "ed25519".to_string() + }; + + let (namespace, rest) = take_array::<32>(rest)?; + let (public_key, rest) = take_array::<32>(rest)?; + Ok(( + ManifestSigner { + signature, + namespace, + public_key, + }, + rest, + )) + } +} + +impl CompactEncodable for Manifest { + fn encoded_size(&self) -> Result { + Ok(1 // Version + + 1 // hash in one byte + + 1 // type in one byte + + self.signer.encoded_size()?) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = write_slice(&[0], buffer)?; + let rest = if &self.hash == "blake2b" { + write_slice(&[0], rest)? + } else { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown hash: {}", &self.hash), + )); + }; + let rest = write_slice(&[1], rest)?; + self.signer.encoded_bytes(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([version], rest) = take_array::<1>(buffer)?; + if version != 0 { + panic!("Unknown manifest version {}", version); + } + let ([hash_id], rest) = take_array::<1>(rest)?; + let hash: String = if hash_id != 0 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown hash id: {hash_id}"), + )); + } else { + "blake2b".to_string() + }; + let ([manifest_type], rest) = take_array::<1>(rest)?; + if manifest_type != 1 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown manifest type: {manifest_type}"), + )); + } + let (signer, rest) = ManifestSigner::decode(rest)?; + Ok((Manifest { hash, signer }, rest)) + } +} From 1771cef52b1a6664e2c6880bb6260e9c95c79fda Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 8 Apr 2025 12:24:22 -0500 Subject: [PATCH 03/32] use macros for encode size/bytes --- src/encoding.rs | 86 ++++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index a327160..60a1e37 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -373,14 +373,36 @@ impl CompactEncoding for State { } } +macro_rules! sum_encoded_size { + // Base case: single field + ($self:ident, $field:ident) => { + $self.$field.encoded_size()? + }; + // Recursive case: first field + rest + ($self: ident, $first:ident, $($rest:ident),+) => { + $self.$first.encoded_size()? + sum_encoded_size!($self, $($rest),+) + }; +} + +macro_rules! chain_encoded_bytes { + // Base case: single field + ($self:ident, $buffer:ident, $field:ident) => { + $self.$field.encoded_bytes($buffer)? + }; + // Recursive case: first field + rest + ($self: ident, $buffer:ident, $first:ident, $($rest:ident),+) => {{ + let rest = $self.$first.encoded_bytes($buffer)?; + chain_encoded_bytes!($self, rest, $($rest),+) + }}; +} + impl CompactEncodable for Node { fn encoded_size(&self) -> Result { - Ok(self.index.encoded_size()? + self.length.encoded_size()? + 32) + Ok(sum_encoded_size!(self, index, length) + 32) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.index.encoded_bytes(buffer)?; - let rest = self.length.encoded_bytes(rest)?; + let rest = chain_encoded_bytes!(self, buffer, index, length); bytes_fixed_from_vec::<32>(&self.hash)?.encoded_bytes(rest) } @@ -407,15 +429,14 @@ impl VecEncodable for Node { Ok(out) } } + impl CompactEncodable for RequestBlock { fn encoded_size(&self) -> Result { - Ok(self.index.encoded_size()? + self.nodes.encoded_size()?) + Ok(sum_encoded_size!(self, index, nodes)) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.index.encoded_bytes(buffer)?; - let rest = self.nodes.encoded_bytes(rest)?; - Ok(rest) + Ok(chain_encoded_bytes!(self, buffer, index, nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -448,13 +469,11 @@ impl CompactEncodable for RequestSeek { impl CompactEncodable for RequestUpgrade { fn encoded_size(&self) -> Result { - Ok(self.start.encoded_size()? + self.length.encoded_size()?) + Ok(sum_encoded_size!(self, start, length)) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.start.encoded_bytes(buffer)?; - let rest = self.length.encoded_bytes(rest)?; - Ok(rest) + Ok(chain_encoded_bytes!(self, buffer, start, length)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -469,14 +488,11 @@ impl CompactEncodable for RequestUpgrade { impl CompactEncodable for DataBlock { fn encoded_size(&self) -> Result { - Ok(self.index.encoded_size()? + self.value.encoded_size()? + self.nodes.encoded_size()?) + Ok(sum_encoded_size!(self, index, value, nodes)) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.index.encoded_bytes(buffer)?; - let rest = self.value.encoded_bytes(rest)?; - let rest = self.nodes.encoded_bytes(rest)?; - Ok(rest) + Ok(chain_encoded_bytes!(self, buffer, index, value, nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -499,13 +515,11 @@ impl CompactEncodable for DataBlock { impl CompactEncodable for DataHash { fn encoded_size(&self) -> Result { - Ok(self.index.encoded_size()? + self.nodes.encoded_size()?) + Ok(sum_encoded_size!(self, index, nodes)) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.index.encoded_bytes(buffer)?; - let rest = self.nodes.encoded_bytes(rest)?; - Ok(rest) + Ok(chain_encoded_bytes!(self, buffer, index, nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -520,13 +534,11 @@ impl CompactEncodable for DataHash { impl CompactEncodable for DataSeek { fn encoded_size(&self) -> Result { - Ok(self.bytes.encoded_size()? + self.nodes.encoded_size()?) + Ok(sum_encoded_size!(self, bytes, nodes)) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.bytes.encoded_bytes(buffer)?; - let rest = self.nodes.encoded_bytes(rest)?; - Ok(rest) + Ok(chain_encoded_bytes!(self, buffer, bytes, nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -539,17 +551,6 @@ impl CompactEncodable for DataSeek { } } -macro_rules! sum_encoded_size { - // Base case: single field - ($self:ident, $field:ident) => { - $self.$field.encoded_size()? - }; - // Recursive case: first field + rest - ($self: ident, $first:ident, $($rest:ident),+) => { - $self.$first.encoded_size()? + sum_encoded_size!($self, $($rest),+) - }; -} - impl CompactEncodable for DataUpgrade { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!( @@ -563,12 +564,15 @@ impl CompactEncodable for DataUpgrade { } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = self.start.encoded_bytes(buffer)?; - let rest = self.length.encoded_bytes(rest)?; - let rest = self.nodes.encoded_bytes(rest)?; - let rest = self.additional_nodes.encoded_bytes(rest)?; - let rest = self.signature.encoded_bytes(rest)?; - Ok(rest) + Ok(chain_encoded_bytes!( + self, + buffer, + start, + length, + nodes, + additional_nodes, + signature + )) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> From a4551a64393005e2359bc07788dd6d8c157ad84f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 9 Apr 2025 11:31:47 -0500 Subject: [PATCH 04/32] add CompactEncodable's to oplog/header --- src/encoding.rs | 26 ++++++ src/oplog/header.rs | 202 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 224 insertions(+), 4 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index 60a1e37..0100c80 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -373,6 +373,9 @@ impl CompactEncoding for State { } } +#[macro_export] +/// Used for defining CompactEncodable::encoded_size. +/// Pass self and a list of fields to call encoded_size on macro_rules! sum_encoded_size { // Base case: single field ($self:ident, $field:ident) => { @@ -384,6 +387,10 @@ macro_rules! sum_encoded_size { }; } +#[macro_export] +// TODO is this exported from the crate? +/// Used for defining CompactEncodable::encoded_bytes. +/// Pass self, the buffer and a list of fields to call encoded_size on macro_rules! chain_encoded_bytes { // Base case: single field ($self:ident, $buffer:ident, $field:ident) => { @@ -694,3 +701,22 @@ impl CompactEncodable for Manifest { Ok((Manifest { hash, signer }, rest)) } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn cmp_node_enc() -> Result<(), EncodingError> { + let node = Node::new(1, vec![4; 32], 66); + let my_buf = CompactEncodable::to_bytes(&node)?; + let mut state = HypercoreState::new(); + + state.preencode(&node)?; + assert_eq!(my_buf.len(), state.end()); + let mut buf = vec![0; state.end()]; + state.encode(&node, &mut buf)?; + assert_eq!(my_buf, buf); + Ok(()) + } +} diff --git a/src/oplog/header.rs b/src/oplog/header.rs index aa27dce..87b8671 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,12 +1,14 @@ -use compact_encoding::EncodingErrorKind; -use compact_encoding::{CompactEncoding, EncodingError, State}; +use compact_encoding::{ + types::{take_array, usize_decode, write_array, CompactEncodable}, + CompactEncoding, EncodingError, EncodingErrorKind, State, +}; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use std::convert::TryInto; use crate::crypto::default_signer_manifest; use crate::crypto::Manifest; -use crate::PartialKeypair; -use crate::VerifyingKey; +use crate::{chain_encoded_bytes, VerifyingKey}; +use crate::{sum_encoded_size, PartialKeypair}; /// Oplog header. #[derive(Debug, Clone)] @@ -81,6 +83,52 @@ impl HeaderTree { } } +macro_rules! decode { + // Match the pattern: decode!(StructName, buffer, {field1: type1, field2: type2, ...}) + ($struct_name:ident, $buffer:expr, { + $($field_name:ident : $field_type:ty),* $(,)? + }) => {{ + + // Variable to hold the current buffer state + let mut current_buffer = $buffer; + + // Decode each field in sequence + $( + let ($field_name, new_buffer) = <$field_type>::decode(current_buffer)?; + current_buffer = new_buffer; + )* + + // Create the struct with decoded fields + let result = $struct_name { + $( + $field_name, + )* + }; + + // Return the struct and the remaining buffer + Ok((result, current_buffer)) + }}; + } + +impl CompactEncodable for HeaderTree { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self, fork, length, root_hash, signature)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(chain_encoded_bytes!( + self, buffer, fork, length, root_hash, signature + )) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode!(HeaderTree, buffer, {fork: u64, length: u64, root_hash: Box<[u8]>, signature: Box<[u8]>}) + } +} + impl CompactEncoding for State { fn preencode(&mut self, value: &HeaderTree) -> Result { self.preencode(&value.fork)?; @@ -110,6 +158,67 @@ impl CompactEncoding for State { } } +impl CompactEncodable for PartialKeypair { + fn encoded_size(&self) -> Result { + Ok(1 // len of public key + + PUBLIC_KEY_LENGTH // public key bytes + + match self.secret { + // Secret key contains the public key + Some(_) => 1 + SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH, + None => 1, + }) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let public_key = self.public.as_bytes().to_vec(); + let rest = public_key.encoded_bytes(buffer)?; + match &self.secret { + Some(sk) => { + let sk_bytes = [&sk.to_bytes()[..], &public_key[..]].concat(); + sk_bytes.encoded_bytes(rest) + } + None => write_array(&[0], rest), + } + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (pk_len, rest) = usize_decode(buffer)?; + let (public, rest) = match pk_len { + PUBLIC_KEY_LENGTH => { + let (pk_bytes, rest) = take_array::(rest)?; + let public = VerifyingKey::from_bytes(&pk_bytes).map_err(|e| { + EncodingError::invalid_data(&format!( + "Could not decode public key. error: [{e}]" + )) + })?; + (public, rest) + } + len => { + return Err(EncodingError::invalid_data(&format!( + "Incorrect secret key length while decoding. length = [{len}]" + ))) + } + }; + let (sk_len, rest) = usize_decode(rest)?; + let (secret, rest) = match sk_len { + 0 => (None, rest), + SECRET_KEY_LENGTH => { + let (sk_bytes, rest) = take_array::(rest)?; + (Some(SigningKey::from_bytes(&sk_bytes)), rest) + } + len => { + return Err(EncodingError::invalid_data(&format!( + "Incorrect secret key length while decoding. length = [{len}]" + ))) + } + }; + Ok((PartialKeypair { public, secret }, rest)) + } +} + /// NB: In Javascript's sodium the secret key contains in itself also the public key, so to /// maintain binary compatibility, we store the public key in the oplog now twice. impl CompactEncoding for State { @@ -171,6 +280,28 @@ pub(crate) struct HeaderHints { pub(crate) contiguous_length: u64, } +impl CompactEncodable for HeaderHints { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self, reorgs, contiguous_length)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(chain_encoded_bytes!( + self, + buffer, + reorgs, + contiguous_length + )) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode!(HeaderHints, buffer, {reorgs: Vec, contiguous_length: u64 }) + } +} + impl CompactEncoding for State { fn preencode(&mut self, value: &HeaderHints) -> Result { self.preencode(&value.reorgs)?; @@ -190,6 +321,47 @@ impl CompactEncoding for State { } } +impl CompactEncodable for Header { + fn encoded_size(&self) -> Result { + Ok(1 + 1 + 32 + sum_encoded_size!(self, manifest, key_pair, user_data, tree, hints)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = write_array(&[1, 2 | 4], buffer)?; + let rest = self.key.encoded_bytes(rest)?; + let rest = self.manifest.encoded_bytes(rest)?; + let rest = self.key_pair.encoded_bytes(rest)?; + let rest = self.user_data.encoded_bytes(rest)?; + let rest = self.tree.encoded_bytes(rest)?; + let rest = self.hints.encoded_bytes(rest)?; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([_version, _flags], rest) = take_array::<2>(buffer)?; + let (key, rest) = take_array::<32>(rest)?; + let (manifest, rest) = Manifest::decode(rest)?; + let (key_pair, rest) = PartialKeypair::decode(rest)?; + let (user_data, rest) = >::decode(rest)?; + let (tree, rest) = HeaderTree::decode(rest)?; + let (hints, rest) = HeaderHints::decode(rest)?; + Ok(( + Header { + key, + manifest, + key_pair, + user_data, + tree, + hints, + }, + rest, + )) + } +} + impl CompactEncoding
for State { fn preencode(&mut self, value: &Header) -> Result { self.add_end(1)?; // Version @@ -293,6 +465,28 @@ mod tests { Ok(()) } + #[test] + fn encode_tree_cmp() -> Result<(), EncodingError> { + let mut enc_state = State::new(); + let tree = HeaderTree { + fork: 520, + length: 647, + root_hash: vec![12; 464].into_boxed_slice(), + signature: vec![46; 22].into_boxed_slice(), + }; + enc_state.preencode(&tree)?; + //let mut buffer = enc_state.create_buffer(); + let mut buffer = vec![0; enc_state.end()]; + enc_state.encode(&tree, &mut buffer)?; + let mut buf2 = vec![0; tree.encoded_size()?]; + assert_eq!(buffer.len(), buf2.len()); + tree.encoded_bytes(&mut buf2)?; + assert_eq!(buffer, buf2); + + //assert_eq!(tree, tree_ret); + Ok(()) + } + #[test] fn encode_header() -> Result<(), EncodingError> { let mut enc_state = State::new(); From c9c45d72880d29a72ebe4be9fae6435d23558c1e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 9 Apr 2025 12:28:41 -0500 Subject: [PATCH 05/32] add entry OplogEntry CompactEncobable --- src/oplog/entry.rs | 134 ++++++++++++++++++++++++++++++++++++++++++++ src/oplog/header.rs | 2 + 2 files changed, 136 insertions(+) diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index e368100..eac8fb6 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,4 +1,7 @@ +use compact_encoding::types::{take_array, take_array_mut, write_array, CompactEncodable}; + use crate::encoding::{CompactEncoding, EncodingError, HypercoreState}; +use crate::{chain_encoded_bytes, decode, sum_encoded_size}; use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade @@ -10,6 +13,24 @@ pub(crate) struct EntryTreeUpgrade { pub(crate) signature: Box<[u8]>, } +impl CompactEncodable for EntryTreeUpgrade { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self, fork, ancestors, length, signature)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(chain_encoded_bytes!( + self, buffer, fork, ancestors, length, signature + )) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode!(EntryTreeUpgrade, buffer, {fork: u64, ancestors: u64, length: u64, signature: Box<[u8]>}) + } +} impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &EntryTreeUpgrade) -> Result { self.0.preencode(&value.fork)?; @@ -43,6 +64,35 @@ impl CompactEncoding for HypercoreState { } } +impl CompactEncodable for BitfieldUpdate { + fn encoded_size(&self) -> Result { + Ok(1 + sum_encoded_size!(self, start, length)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let drop = if self.drop { 1 } else { 0 }; + let rest = write_array(&[drop], buffer)?; + Ok(chain_encoded_bytes!(self, rest, start, length)) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([flags], rest) = take_array::<1>(buffer)?; + let (start, rest) = u64::decode(rest)?; + let (length, rest) = u64::decode(rest)?; + Ok(( + BitfieldUpdate { + drop: flags == 1, + start, + length, + }, + rest, + )) + } +} + impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &BitfieldUpdate) -> Result { self.0.add_end(1)?; @@ -83,6 +133,90 @@ pub struct Entry { pub(crate) bitfield: Option, } +impl CompactEncodable for Entry { + fn encoded_size(&self) -> Result { + let mut out = 1; // flags + if !self.user_data.is_empty() { + out += self.user_data.encoded_size()?; + } + if !self.tree_nodes.is_empty() { + out += self.tree_nodes.encoded_size()?; + } + if let Some(tree_upgrade) = &self.tree_upgrade { + out += tree_upgrade.encoded_size()?; + } + if let Some(bitfield) = &self.bitfield { + out += bitfield.encoded_size()?; + } + Ok(out) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let (flag_buf, mut rest) = take_array_mut::<1>(buffer)?; + let mut flags = 0u8; + if !self.user_data.is_empty() { + flags |= 1; + rest = self.user_data.encoded_bytes(rest)?; + } + if !self.tree_nodes.is_empty() { + flags |= 2; + rest = self.tree_nodes.encoded_bytes(rest)?; + } + if let Some(tree_upgrade) = &self.tree_upgrade { + flags |= 4; + rest = tree_upgrade.encoded_bytes(rest)?; + } + if let Some(bitfield) = &self.bitfield { + flags |= 8; + rest = bitfield.encoded_bytes(rest)?; + } + flag_buf[0] = flags; + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([flags], rest) = take_array::<1>(buffer)?; + let (user_data, rest) = if flags & 1 != 0 { + >::decode(rest)? + } else { + (Default::default(), rest) + }; + + let (tree_nodes, rest) = if flags & 2 != 0 { + >::decode(buffer)? + } else { + (Default::default(), rest) + }; + + let (tree_upgrade, rest) = if flags & 2 != 0 { + let (x, rest) = EntryTreeUpgrade::decode(buffer)?; + (Some(x), rest) + } else { + (Default::default(), rest) + }; + + let (bitfield, rest) = if flags & 2 != 0 { + let (x, rest) = BitfieldUpdate::decode(buffer)?; + (Some(x), rest) + } else { + (Default::default(), rest) + }; + + Ok(( + Self { + user_data, + tree_nodes, + tree_upgrade, + bitfield, + }, + rest, + )) + } +} + impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &Entry) -> Result { self.0.add_end(1)?; // flags diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 87b8671..3060c91 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -83,6 +83,8 @@ impl HeaderTree { } } +#[macro_export] +/// Helper for decoding a struct with compact encodable macro_rules! decode { // Match the pattern: decode!(StructName, buffer, {field1: type1, field2: type2, ...}) ($struct_name:ident, $buffer:expr, { From a966da5515bcbd03c2e06a47c9b9784138aecbfb Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 17:18:54 -0400 Subject: [PATCH 06/32] encoded_bytes changed to encode --- src/encoding.rs | 42 ++++++++++++++++++++++-------------------- src/oplog/entry.rs | 16 ++++++++-------- src/oplog/header.rs | 28 ++++++++++++++-------------- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index 0100c80..713eb1f 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,9 +1,11 @@ //! Hypercore-specific compact encodings -use compact_encoding::encodable::{ - bytes_fixed_from_vec, encode_bytes_fixed, write_slice, VecEncodable, +pub use compact_encoding::{ + encodable::{ + bytes_fixed_from_vec, encode_bytes_fixed, take_array, take_array_mut, usize_encoded_size, + write_array, write_slice, CompactEncodable, VecEncodable, + }, + CompactEncoding, EncodingError, EncodingErrorKind, State, }; -use compact_encoding::types::{take_array, usize_encoded_size, CompactEncodable}; -pub use compact_encoding::{CompactEncoding, EncodingError, EncodingErrorKind, State}; use std::convert::TryInto; use std::ops::{Deref, DerefMut}; @@ -389,16 +391,16 @@ macro_rules! sum_encoded_size { #[macro_export] // TODO is this exported from the crate? -/// Used for defining CompactEncodable::encoded_bytes. +/// Used for defining CompactEncodable::encode. /// Pass self, the buffer and a list of fields to call encoded_size on macro_rules! chain_encoded_bytes { // Base case: single field ($self:ident, $buffer:ident, $field:ident) => { - $self.$field.encoded_bytes($buffer)? + $self.$field.encode($buffer)? }; // Recursive case: first field + rest ($self: ident, $buffer:ident, $first:ident, $($rest:ident),+) => {{ - let rest = $self.$first.encoded_bytes($buffer)?; + let rest = $self.$first.encode($buffer)?; chain_encoded_bytes!($self, rest, $($rest),+) }}; } @@ -408,9 +410,9 @@ impl CompactEncodable for Node { Ok(sum_encoded_size!(self, index, length) + 32) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let rest = chain_encoded_bytes!(self, buffer, index, length); - bytes_fixed_from_vec::<32>(&self.hash)?.encoded_bytes(rest) + bytes_fixed_from_vec::<32>(&self.hash)?.encode(rest) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -442,7 +444,7 @@ impl CompactEncodable for RequestBlock { Ok(sum_encoded_size!(self, index, nodes)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!(self, buffer, index, nodes)) } @@ -461,8 +463,8 @@ impl CompactEncodable for RequestSeek { self.bytes.encoded_size() } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - self.bytes.encoded_bytes(buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + self.bytes.encode(buffer) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -479,7 +481,7 @@ impl CompactEncodable for RequestUpgrade { Ok(sum_encoded_size!(self, start, length)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!(self, buffer, start, length)) } @@ -498,7 +500,7 @@ impl CompactEncodable for DataBlock { Ok(sum_encoded_size!(self, index, value, nodes)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!(self, buffer, index, value, nodes)) } @@ -525,7 +527,7 @@ impl CompactEncodable for DataHash { Ok(sum_encoded_size!(self, index, nodes)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!(self, buffer, index, nodes)) } @@ -544,7 +546,7 @@ impl CompactEncodable for DataSeek { Ok(sum_encoded_size!(self, bytes, nodes)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!(self, buffer, bytes, nodes)) } @@ -570,7 +572,7 @@ impl CompactEncodable for DataUpgrade { )) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!( self, buffer, @@ -611,7 +613,7 @@ impl CompactEncodable for ManifestSigner { ) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let rest = if &self.signature == "ed25519" { write_slice(&[0], buffer)? } else { @@ -659,7 +661,7 @@ impl CompactEncodable for Manifest { + self.signer.encoded_size()?) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let rest = write_slice(&[0], buffer)?; let rest = if &self.hash == "blake2b" { write_slice(&[0], rest)? @@ -670,7 +672,7 @@ impl CompactEncodable for Manifest { )); }; let rest = write_slice(&[1], rest)?; - self.signer.encoded_bytes(rest) + self.signer.encode(rest) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index eac8fb6..9548d61 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,4 +1,4 @@ -use compact_encoding::types::{take_array, take_array_mut, write_array, CompactEncodable}; +use compact_encoding::encodable::{take_array, take_array_mut, write_array, CompactEncodable}; use crate::encoding::{CompactEncoding, EncodingError, HypercoreState}; use crate::{chain_encoded_bytes, decode, sum_encoded_size}; @@ -18,7 +18,7 @@ impl CompactEncodable for EntryTreeUpgrade { Ok(sum_encoded_size!(self, fork, ancestors, length, signature)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!( self, buffer, fork, ancestors, length, signature )) @@ -69,7 +69,7 @@ impl CompactEncodable for BitfieldUpdate { Ok(1 + sum_encoded_size!(self, start, length)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let drop = if self.drop { 1 } else { 0 }; let rest = write_array(&[drop], buffer)?; Ok(chain_encoded_bytes!(self, rest, start, length)) @@ -151,24 +151,24 @@ impl CompactEncodable for Entry { Ok(out) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let (flag_buf, mut rest) = take_array_mut::<1>(buffer)?; let mut flags = 0u8; if !self.user_data.is_empty() { flags |= 1; - rest = self.user_data.encoded_bytes(rest)?; + rest = self.user_data.encode(rest)?; } if !self.tree_nodes.is_empty() { flags |= 2; - rest = self.tree_nodes.encoded_bytes(rest)?; + rest = self.tree_nodes.encode(rest)?; } if let Some(tree_upgrade) = &self.tree_upgrade { flags |= 4; - rest = tree_upgrade.encoded_bytes(rest)?; + rest = tree_upgrade.encode(rest)?; } if let Some(bitfield) = &self.bitfield { flags |= 8; - rest = bitfield.encoded_bytes(rest)?; + rest = bitfield.encode(rest)?; } flag_buf[0] = flags; Ok(rest) diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 3060c91..1ff1a17 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,5 +1,5 @@ use compact_encoding::{ - types::{take_array, usize_decode, write_array, CompactEncodable}, + encodable::{take_array, usize_decode, write_array, CompactEncodable}, CompactEncoding, EncodingError, EncodingErrorKind, State, }; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; @@ -117,7 +117,7 @@ impl CompactEncodable for HeaderTree { Ok(sum_encoded_size!(self, fork, length, root_hash, signature)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!( self, buffer, fork, length, root_hash, signature )) @@ -171,13 +171,13 @@ impl CompactEncodable for PartialKeypair { }) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let public_key = self.public.as_bytes().to_vec(); - let rest = public_key.encoded_bytes(buffer)?; + let rest = public_key.encode(buffer)?; match &self.secret { Some(sk) => { let sk_bytes = [&sk.to_bytes()[..], &public_key[..]].concat(); - sk_bytes.encoded_bytes(rest) + sk_bytes.encode(rest) } None => write_array(&[0], rest), } @@ -287,7 +287,7 @@ impl CompactEncodable for HeaderHints { Ok(sum_encoded_size!(self, reorgs, contiguous_length)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { Ok(chain_encoded_bytes!( self, buffer, @@ -328,14 +328,14 @@ impl CompactEncodable for Header { Ok(1 + 1 + 32 + sum_encoded_size!(self, manifest, key_pair, user_data, tree, hints)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let rest = write_array(&[1, 2 | 4], buffer)?; - let rest = self.key.encoded_bytes(rest)?; - let rest = self.manifest.encoded_bytes(rest)?; - let rest = self.key_pair.encoded_bytes(rest)?; - let rest = self.user_data.encoded_bytes(rest)?; - let rest = self.tree.encoded_bytes(rest)?; - let rest = self.hints.encoded_bytes(rest)?; + let rest = self.key.encode(rest)?; + let rest = self.manifest.encode(rest)?; + let rest = self.key_pair.encode(rest)?; + let rest = self.user_data.encode(rest)?; + let rest = self.tree.encode(rest)?; + let rest = self.hints.encode(rest)?; Ok(rest) } @@ -482,7 +482,7 @@ mod tests { enc_state.encode(&tree, &mut buffer)?; let mut buf2 = vec![0; tree.encoded_size()?]; assert_eq!(buffer.len(), buf2.len()); - tree.encoded_bytes(&mut buf2)?; + tree.encode(&mut buf2)?; assert_eq!(buffer, buf2); //assert_eq!(tree, tree_ret); From a17a8e4360a6d6bc39179d209af2aa18815a2812 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 16 Apr 2025 15:38:56 -0400 Subject: [PATCH 07/32] WIP Removing State/HypercoreState --- src/encoding.rs | 421 +++--------------------------------------------- 1 file changed, 18 insertions(+), 403 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index 713eb1f..fb67b6f 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,382 +1,16 @@ //! Hypercore-specific compact encodings -pub use compact_encoding::{ - encodable::{ - bytes_fixed_from_vec, encode_bytes_fixed, take_array, take_array_mut, usize_encoded_size, - write_array, write_slice, CompactEncodable, VecEncodable, - }, - CompactEncoding, EncodingError, EncodingErrorKind, State, -}; -use std::convert::TryInto; -use std::ops::{Deref, DerefMut}; - use crate::{ crypto::{Manifest, ManifestSigner}, DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, }; - -#[derive(Debug, Clone)] -/// Wrapper struct for compact_encoding::State -pub struct HypercoreState(pub State); - -impl Default for HypercoreState { - /// Passthrought to compact_encoding - fn default() -> Self { - Self::new() - } -} - -impl HypercoreState { - /// Passthrought to compact_encoding - pub fn new() -> HypercoreState { - HypercoreState(State::new()) - } - - /// Passthrought to compact_encoding - pub fn new_with_size(size: usize) -> (HypercoreState, Box<[u8]>) { - let (state, buffer) = State::new_with_size(size); - (HypercoreState(state), buffer) - } - - /// Passthrought to compact_encoding - pub fn new_with_start_and_end(start: usize, end: usize) -> HypercoreState { - HypercoreState(State::new_with_start_and_end(start, end)) - } - - /// Passthrought to compact_encoding - pub fn from_buffer(buffer: &[u8]) -> HypercoreState { - HypercoreState(State::from_buffer(buffer)) - } -} - -impl Deref for HypercoreState { - type Target = State; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for HypercoreState { - fn deref_mut(&mut self) -> &mut State { - &mut self.0 - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &Node) -> Result { - self.0.preencode(&value.index)?; - self.0.preencode(&value.length)?; - self.0.preencode_fixed_32() - } - - fn encode(&mut self, value: &Node, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.0.encode(&value.length, buffer)?; - self.0.encode_fixed_32(&value.hash, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - let hash: Box<[u8]> = self.0.decode_fixed_32(buffer)?; - Ok(Node::new(index, hash.to_vec(), length)) - } -} - -impl CompactEncoding> for HypercoreState { - fn preencode(&mut self, value: &Vec) -> Result { - let len = value.len(); - self.0.preencode(&len)?; - for val in value { - self.preencode(val)?; - } - Ok(self.end()) - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) -> Result { - let len = value.len(); - self.0.encode(&len, buffer)?; - for val in value { - self.encode(val, buffer)?; - } - Ok(self.start()) - } - - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let len: usize = self.0.decode(buffer)?; - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode(buffer)?); - } - Ok(value) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestBlock) -> Result { - self.0.preencode(&value.index)?; - self.0.preencode(&value.nodes) - } - - fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.0.encode(&value.nodes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let nodes: u64 = self.0.decode(buffer)?; - Ok(RequestBlock { index, nodes }) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestSeek) -> Result { - self.0.preencode(&value.bytes) - } - - fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) -> Result { - self.0.encode(&value.bytes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let bytes: u64 = self.0.decode(buffer)?; - Ok(RequestSeek { bytes }) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestUpgrade) -> Result { - self.0.preencode(&value.start)?; - self.0.preencode(&value.length) - } - - fn encode( - &mut self, - value: &RequestUpgrade, - buffer: &mut [u8], - ) -> Result { - self.0.encode(&value.start, buffer)?; - self.0.encode(&value.length, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let start: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - Ok(RequestUpgrade { start, length }) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataBlock) -> Result { - self.0.preencode(&value.index)?; - self.0.preencode(&value.value)?; - self.preencode(&value.nodes) - } - - fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.0.encode(&value.value, buffer)?; - self.encode(&value.nodes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let value: Vec = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - Ok(DataBlock { - index, - value, - nodes, - }) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataHash) -> Result { - self.0.preencode(&value.index)?; - self.preencode(&value.nodes) - } - - fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.encode(&value.nodes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - Ok(DataHash { index, nodes }) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataSeek) -> Result { - self.0.preencode(&value.bytes)?; - self.preencode(&value.nodes) - } - - fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) -> Result { - self.0.encode(&value.bytes, buffer)?; - self.encode(&value.nodes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let bytes: u64 = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - Ok(DataSeek { bytes, nodes }) - } -} - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataUpgrade) -> Result { - self.0.preencode(&value.start)?; - self.0.preencode(&value.length)?; - self.preencode(&value.nodes)?; - self.preencode(&value.additional_nodes)?; - self.0.preencode(&value.signature) - } - - fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) -> Result { - self.0.encode(&value.start, buffer)?; - self.0.encode(&value.length, buffer)?; - self.encode(&value.nodes, buffer)?; - self.encode(&value.additional_nodes, buffer)?; - self.0.encode(&value.signature, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let start: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - let additional_nodes: Vec = self.decode(buffer)?; - let signature: Vec = self.0.decode(buffer)?; - Ok(DataUpgrade { - start, - length, - nodes, - additional_nodes, - signature, - }) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &Manifest) -> Result { - self.add_end(1)?; // Version - self.add_end(1)?; // hash in one byte - self.add_end(1)?; // type in one byte - self.preencode(&value.signer) - } - - fn encode(&mut self, value: &Manifest, buffer: &mut [u8]) -> Result { - self.set_byte_to_buffer(0, buffer)?; // Version - if &value.hash == "blake2b" { - self.set_byte_to_buffer(0, buffer)?; // Version - } else { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown hash: {}", &value.hash), - )); - } - // Type. 0: static, 1: signer, 2: multiple signers - self.set_byte_to_buffer(1, buffer)?; // Version - self.encode(&value.signer, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let version: u8 = self.decode_u8(buffer)?; - if version != 0 { - panic!("Unknown manifest version {}", version); - } - let hash_id: u8 = self.decode_u8(buffer)?; - let hash: String = if hash_id != 0 { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown hash id: {hash_id}"), - )); - } else { - "blake2b".to_string() - }; - - let manifest_type: u8 = self.decode_u8(buffer)?; - if manifest_type != 1 { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown manifest type: {manifest_type}"), - )); - } - let signer: ManifestSigner = self.decode(buffer)?; - - Ok(Manifest { hash, signer }) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, _value: &ManifestSigner) -> Result { - self.add_end(1)?; // Signature - self.preencode_fixed_32()?; - self.preencode_fixed_32() - } - - fn encode( - &mut self, - value: &ManifestSigner, - buffer: &mut [u8], - ) -> Result { - if &value.signature == "ed25519" { - self.set_byte_to_buffer(0, buffer)?; - } else { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown signature type: {}", &value.signature), - )); - } - self.encode_fixed_32(&value.namespace, buffer)?; - self.encode_fixed_32(&value.public_key, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let signature_id: u8 = self.decode_u8(buffer)?; - let signature: String = if signature_id != 0 { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown signature id: {signature_id}"), - )); - } else { - "ed25519".to_string() - }; - let namespace: [u8; 32] = - self.decode_fixed_32(buffer)? - .to_vec() - .try_into() - .map_err(|_err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - "Invalid namespace in manifest signer", - ) - })?; - let public_key: [u8; 32] = - self.decode_fixed_32(buffer)? - .to_vec() - .try_into() - .map_err(|_err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - "Invalid public key in manifest signer", - ) - })?; - - Ok(ManifestSigner { - signature, - namespace, - public_key, - }) - } -} +use compact_encoding::encoded_size_usize; +pub use compact_encoding::{ + bytes_fixed_from_vec, encode_bytes_fixed, take_array, take_array_mut, write_array, write_slice, + CompactEncoding, EncodingError, EncodingErrorKind, VecEncodable, +}; #[macro_export] -/// Used for defining CompactEncodable::encoded_size. +/// Used for defining CompactEncoding::encoded_size. /// Pass self and a list of fields to call encoded_size on macro_rules! sum_encoded_size { // Base case: single field @@ -391,7 +25,7 @@ macro_rules! sum_encoded_size { #[macro_export] // TODO is this exported from the crate? -/// Used for defining CompactEncodable::encode. +/// Used for defining CompactEncoding::encode. /// Pass self, the buffer and a list of fields to call encoded_size on macro_rules! chain_encoded_bytes { // Base case: single field @@ -405,7 +39,7 @@ macro_rules! chain_encoded_bytes { }}; } -impl CompactEncodable for Node { +impl CompactEncoding for Node { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, index, length) + 32) } @@ -431,7 +65,7 @@ impl VecEncodable for Node { where Self: Sized, { - let mut out = usize_encoded_size(vec.len()); + let mut out = encoded_size_usize(vec.len()); for x in vec { out += x.encoded_size()?; } @@ -439,7 +73,7 @@ impl VecEncodable for Node { } } -impl CompactEncodable for RequestBlock { +impl CompactEncoding for RequestBlock { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, index, nodes)) } @@ -458,7 +92,7 @@ impl CompactEncodable for RequestBlock { } } -impl CompactEncodable for RequestSeek { +impl CompactEncoding for RequestSeek { fn encoded_size(&self) -> Result { self.bytes.encoded_size() } @@ -476,7 +110,7 @@ impl CompactEncodable for RequestSeek { } } -impl CompactEncodable for RequestUpgrade { +impl CompactEncoding for RequestUpgrade { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, start, length)) } @@ -495,7 +129,7 @@ impl CompactEncodable for RequestUpgrade { } } -impl CompactEncodable for DataBlock { +impl CompactEncoding for DataBlock { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, index, value, nodes)) } @@ -522,7 +156,7 @@ impl CompactEncodable for DataBlock { } } -impl CompactEncodable for DataHash { +impl CompactEncoding for DataHash { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, index, nodes)) } @@ -541,7 +175,7 @@ impl CompactEncodable for DataHash { } } -impl CompactEncodable for DataSeek { +impl CompactEncoding for DataSeek { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, bytes, nodes)) } @@ -560,7 +194,7 @@ impl CompactEncodable for DataSeek { } } -impl CompactEncodable for DataUpgrade { +impl CompactEncoding for DataUpgrade { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!( self, @@ -606,7 +240,7 @@ impl CompactEncodable for DataUpgrade { } } -impl CompactEncodable for ManifestSigner { +impl CompactEncoding for ManifestSigner { fn encoded_size(&self) -> Result { Ok( 1 /* Signature */ + 32 /* namespace */ + 32, /* public_key */ @@ -653,7 +287,7 @@ impl CompactEncodable for ManifestSigner { } } -impl CompactEncodable for Manifest { +impl CompactEncoding for Manifest { fn encoded_size(&self) -> Result { Ok(1 // Version + 1 // hash in one byte @@ -703,22 +337,3 @@ impl CompactEncodable for Manifest { Ok((Manifest { hash, signer }, rest)) } } - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn cmp_node_enc() -> Result<(), EncodingError> { - let node = Node::new(1, vec![4; 32], 66); - let my_buf = CompactEncodable::to_bytes(&node)?; - let mut state = HypercoreState::new(); - - state.preencode(&node)?; - assert_eq!(my_buf.len(), state.end()); - let mut buf = vec![0; state.end()]; - state.encode(&node, &mut buf)?; - assert_eq!(my_buf, buf); - Ok(()) - } -} From dffe4dd95beb4cd2e4c847c28b79a1fa96a9ed12 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 17 Apr 2025 14:17:31 -0400 Subject: [PATCH 08/32] rework crypto/hash with new cenc --- src/crypto/hash.rs | 47 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index f744048..d8f7e34 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -3,7 +3,9 @@ use blake2::{ Blake2b, Blake2bMac, Digest, }; use byteorder::{BigEndian, WriteBytesExt}; -use compact_encoding::State; +use compact_encoding::{ + as_array, to_encoded_bytes, CompactEncoding, EncodingError, FixedWidthEncoding, +}; use ed25519_dalek::VerifyingKey; use merkle_tree_stream::Node as NodeTrait; use std::convert::AsRef; @@ -124,10 +126,9 @@ impl Hash { /// Hash data pub(crate) fn data(data: &[u8]) -> Self { - let (mut state, mut size) = State::new_with_size(8); - state - .encode_u64(data.len() as u64, &mut size) - .expect("Encoding u64 should not fail"); + let size = + (|| Ok::<_, EncodingError>(to_encoded_bytes!((data.len() as u64).as_fixed_width())))() + .expect("Encoding u64 should not fail"); let mut hasher = Blake2b256::new(); hasher.update(LEAF_TYPE); @@ -147,9 +148,8 @@ impl Hash { (right, left) }; - let (mut state, mut size) = State::new_with_size(8); - state - .encode_u64(node1.length + node2.length, &mut size) + let len = node1.length + node2.length; + let size: Vec = (|| Ok::<_, EncodingError>(to_encoded_bytes!(len.as_fixed_width())))() .expect("Encoding u64 should not fail"); let mut hasher = Blake2b256::new(); @@ -170,12 +170,7 @@ impl Hash { for node in roots { let node = node.as_ref(); - let (mut state, mut buffer) = State::new_with_size(16); - state - .encode_u64(node.index(), &mut buffer) - .expect("Encoding u64 should not fail"); - state - .encode_u64(node.len(), &mut buffer) + let buffer = (|| Ok::<_, EncodingError>(to_encoded_bytes!(node.index(), node.len())))() .expect("Encoding u64 should not fail"); hasher.update(node.hash()); @@ -212,20 +207,16 @@ impl DerefMut for Hash { /// Create a signable buffer for tree. This is treeSignable in Javascript. /// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 pub(crate) fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { - let (mut state, mut buffer) = State::new_with_size(80); - state - .encode_fixed_32(&TREE, &mut buffer) - .expect("Should be able "); - state - .encode_fixed_32(hash, &mut buffer) - .expect("Encoding fixed 32 bytes should not fail"); - state - .encode_u64(length, &mut buffer) - .expect("Encoding u64 should not fail"); - state - .encode_u64(fork, &mut buffer) - .expect("Encoding u64 should not fail"); - buffer + (|| { + Ok::<_, EncodingError>(to_encoded_bytes!( + &TREE, + as_array::<32>(hash)?, + length.as_fixed_width(), + fork.as_fixed_width() + )) + })() + .expect("Encoding should not fail") + .into() } #[cfg(test)] From c4170f4b7842738ee7cb2baadcd50e234729eef4 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 17 Apr 2025 14:18:40 -0400 Subject: [PATCH 09/32] rework tree/merkel_tree with new cenc --- src/tree/merkle_tree.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index c957919..e7bca4d 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,4 +1,6 @@ -use compact_encoding::State; +use compact_encoding::{ + as_array, map_decode, to_encoded_bytes, EncodingError, FixedWidthEncoding, FixedWidthU64, +}; use ed25519_dalek::Signature; use futures::future::Either; use intmap::IntMap; @@ -658,13 +660,11 @@ impl MerkleTree { pub(crate) fn flush_nodes(&mut self) -> Vec { let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); for (_, node) in self.unflushed.drain() { - let (mut state, mut buffer) = State::new_with_size(40); - state - .encode_u64(node.length, &mut buffer) - .expect("Encoding u64 should not fail"); - state - .encode_fixed_32(&node.hash, &mut buffer) - .expect("Encoding fixed 32 bytes should not fail"); + let buffer = (|| { + let hash = as_array::<32>(&node.hash)?; + Ok::, EncodingError>(to_encoded_bytes!(node.length.as_fixed_width(), hash)) + })() + .expect("Encoding u64 should not fail"); infos_to_flush.push(StoreInfo::new_content( Store::Tree, node.index * 40, @@ -1475,8 +1475,7 @@ fn index_from_info(info: &StoreInfo) -> u64 { fn node_from_bytes(index: &u64, data: &[u8]) -> Result { let len_buf = &data[..8]; let hash = &data[8..]; - let mut state = State::from_buffer(len_buf); - let len = state.decode_u64(len_buf)?; + let len = map_decode!(len_buf, [FixedWidthU64<'_>]).0 .0; Ok(Node::new(*index, hash.to_vec(), len)) } From fbad5064f75171605bd06f01302932325ab2aa90 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 17 Apr 2025 15:14:22 -0400 Subject: [PATCH 10/32] use new compact encoding --- src/oplog/entry.rs | 151 +------------------------------------ src/oplog/header.rs | 179 +++----------------------------------------- 2 files changed, 13 insertions(+), 317 deletions(-) diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 9548d61..83571de 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,6 +1,5 @@ -use compact_encoding::encodable::{take_array, take_array_mut, write_array, CompactEncodable}; +use compact_encoding::{take_array, take_array_mut, write_array, CompactEncoding, EncodingError}; -use crate::encoding::{CompactEncoding, EncodingError, HypercoreState}; use crate::{chain_encoded_bytes, decode, sum_encoded_size}; use crate::{common::BitfieldUpdate, Node}; @@ -13,7 +12,7 @@ pub(crate) struct EntryTreeUpgrade { pub(crate) signature: Box<[u8]>, } -impl CompactEncodable for EntryTreeUpgrade { +impl CompactEncoding for EntryTreeUpgrade { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, fork, ancestors, length, signature)) } @@ -31,40 +30,8 @@ impl CompactEncodable for EntryTreeUpgrade { decode!(EntryTreeUpgrade, buffer, {fork: u64, ancestors: u64, length: u64, signature: Box<[u8]>}) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &EntryTreeUpgrade) -> Result { - self.0.preencode(&value.fork)?; - self.0.preencode(&value.ancestors)?; - self.0.preencode(&value.length)?; - self.0.preencode(&value.signature) - } - - fn encode( - &mut self, - value: &EntryTreeUpgrade, - buffer: &mut [u8], - ) -> Result { - self.0.encode(&value.fork, buffer)?; - self.0.encode(&value.ancestors, buffer)?; - self.0.encode(&value.length, buffer)?; - self.0.encode(&value.signature, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let fork: u64 = self.0.decode(buffer)?; - let ancestors: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - let signature: Box<[u8]> = self.0.decode(buffer)?; - Ok(EntryTreeUpgrade { - fork, - ancestors, - length, - signature, - }) - } -} -impl CompactEncodable for BitfieldUpdate { +impl CompactEncoding for BitfieldUpdate { fn encoded_size(&self) -> Result { Ok(1 + sum_encoded_size!(self, start, length)) } @@ -93,36 +60,6 @@ impl CompactEncodable for BitfieldUpdate { } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &BitfieldUpdate) -> Result { - self.0.add_end(1)?; - self.0.preencode(&value.start)?; - self.0.preencode(&value.length) - } - - fn encode( - &mut self, - value: &BitfieldUpdate, - buffer: &mut [u8], - ) -> Result { - let flags: u8 = if value.drop { 1 } else { 0 }; - self.0.set_byte_to_buffer(flags, buffer)?; - self.0.encode(&value.start, buffer)?; - self.0.encode(&value.length, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let flags = self.0.decode_u8(buffer)?; - let start: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - Ok(BitfieldUpdate { - drop: flags == 1, - start, - length, - }) - } -} - /// Oplog Entry #[derive(Debug)] pub struct Entry { @@ -133,7 +70,7 @@ pub struct Entry { pub(crate) bitfield: Option, } -impl CompactEncodable for Entry { +impl CompactEncoding for Entry { fn encoded_size(&self) -> Result { let mut out = 1; // flags if !self.user_data.is_empty() { @@ -216,83 +153,3 @@ impl CompactEncodable for Entry { )) } } - -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &Entry) -> Result { - self.0.add_end(1)?; // flags - if !value.user_data.is_empty() { - self.0.preencode(&value.user_data)?; - } - if !value.tree_nodes.is_empty() { - self.preencode(&value.tree_nodes)?; - } - if let Some(tree_upgrade) = &value.tree_upgrade { - self.preencode(tree_upgrade)?; - } - if let Some(bitfield) = &value.bitfield { - self.preencode(bitfield)?; - } - Ok(self.end()) - } - - fn encode(&mut self, value: &Entry, buffer: &mut [u8]) -> Result { - let start = self.0.start(); - self.0.add_start(1)?; - let mut flags: u8 = 0; - if !value.user_data.is_empty() { - flags |= 1; - self.0.encode(&value.user_data, buffer)?; - } - if !value.tree_nodes.is_empty() { - flags |= 2; - self.encode(&value.tree_nodes, buffer)?; - } - if let Some(tree_upgrade) = &value.tree_upgrade { - flags |= 4; - self.encode(tree_upgrade, buffer)?; - } - if let Some(bitfield) = &value.bitfield { - flags |= 8; - self.encode(bitfield, buffer)?; - } - - buffer[start] = flags; - Ok(self.0.start()) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let flags = self.0.decode_u8(buffer)?; - let user_data: Vec = if flags & 1 != 0 { - self.0.decode(buffer)? - } else { - vec![] - }; - - let tree_nodes: Vec = if flags & 2 != 0 { - self.decode(buffer)? - } else { - vec![] - }; - - let tree_upgrade: Option = if flags & 4 != 0 { - let value: EntryTreeUpgrade = self.decode(buffer)?; - Some(value) - } else { - None - }; - - let bitfield: Option = if flags & 8 != 0 { - let value: BitfieldUpdate = self.decode(buffer)?; - Some(value) - } else { - None - }; - - Ok(Entry { - user_data, - tree_nodes, - tree_upgrade, - bitfield, - }) - } -} diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 1ff1a17..56b9a47 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,9 +1,5 @@ -use compact_encoding::{ - encodable::{take_array, usize_decode, write_array, CompactEncodable}, - CompactEncoding, EncodingError, EncodingErrorKind, State, -}; +use compact_encoding::{decode_usize, take_array, write_array, CompactEncoding, EncodingError}; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; -use std::convert::TryInto; use crate::crypto::default_signer_manifest; use crate::crypto::Manifest; @@ -112,7 +108,7 @@ macro_rules! decode { }}; } -impl CompactEncodable for HeaderTree { +impl CompactEncoding for HeaderTree { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, fork, length, root_hash, signature)) } @@ -131,36 +127,9 @@ impl CompactEncodable for HeaderTree { } } -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTree) -> Result { - self.preencode(&value.fork)?; - self.preencode(&value.length)?; - self.preencode(&value.root_hash)?; - self.preencode(&value.signature) - } - - fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) -> Result { - self.encode(&value.fork, buffer)?; - self.encode(&value.length, buffer)?; - self.encode(&value.root_hash, buffer)?; - self.encode(&value.signature, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let fork: u64 = self.decode(buffer)?; - let length: u64 = self.decode(buffer)?; - let root_hash: Box<[u8]> = self.decode(buffer)?; - let signature: Box<[u8]> = self.decode(buffer)?; - Ok(HeaderTree { - fork, - length, - root_hash, - signature, - }) - } -} - -impl CompactEncodable for PartialKeypair { +/// NB: In Javascript's sodium the secret key contains in itself also the public key, so to +/// maintain binary compatibility, we store the public key in the oplog now twice. +impl CompactEncoding for PartialKeypair { fn encoded_size(&self) -> Result { Ok(1 // len of public key + PUBLIC_KEY_LENGTH // public key bytes @@ -187,7 +156,7 @@ impl CompactEncodable for PartialKeypair { where Self: Sized, { - let (pk_len, rest) = usize_decode(buffer)?; + let (pk_len, rest) = decode_usize(buffer)?; let (public, rest) = match pk_len { PUBLIC_KEY_LENGTH => { let (pk_bytes, rest) = take_array::(rest)?; @@ -204,7 +173,7 @@ impl CompactEncodable for PartialKeypair { ))) } }; - let (sk_len, rest) = usize_decode(rest)?; + let (sk_len, rest) = decode_usize(rest)?; let (secret, rest) = match sk_len { 0 => (None, rest), SECRET_KEY_LENGTH => { @@ -221,60 +190,6 @@ impl CompactEncodable for PartialKeypair { } } -/// NB: In Javascript's sodium the secret key contains in itself also the public key, so to -/// maintain binary compatibility, we store the public key in the oplog now twice. -impl CompactEncoding for State { - fn preencode(&mut self, value: &PartialKeypair) -> Result { - self.add_end(1 + PUBLIC_KEY_LENGTH)?; - match &value.secret { - Some(_) => { - // Also add room for the public key - self.add_end(1 + SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH) - } - None => self.add_end(1), - } - } - - fn encode( - &mut self, - value: &PartialKeypair, - buffer: &mut [u8], - ) -> Result { - let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); - self.encode(&public_key_bytes, buffer)?; - match &value.secret { - Some(secret_key) => { - let mut secret_key_bytes: Vec = - Vec::with_capacity(SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH); - secret_key_bytes.extend_from_slice(&secret_key.to_bytes()); - secret_key_bytes.extend_from_slice(&public_key_bytes); - let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); - self.encode(&secret_key_bytes, buffer) - } - None => self.set_byte_to_buffer(0, buffer), - } - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let public_key_bytes: Box<[u8]> = self.decode(buffer)?; - let public_key_bytes: [u8; PUBLIC_KEY_LENGTH] = - public_key_bytes[0..PUBLIC_KEY_LENGTH].try_into().unwrap(); - let secret_key_bytes: Box<[u8]> = self.decode(buffer)?; - let secret: Option = if secret_key_bytes.is_empty() { - None - } else { - let secret_key_bytes: [u8; SECRET_KEY_LENGTH] = - secret_key_bytes[0..SECRET_KEY_LENGTH].try_into().unwrap(); - Some(SigningKey::from_bytes(&secret_key_bytes)) - }; - - Ok(PartialKeypair { - public: VerifyingKey::from_bytes(&public_key_bytes).unwrap(), - secret, - }) - } -} - /// Oplog header hints #[derive(Debug, Clone)] pub(crate) struct HeaderHints { @@ -282,7 +197,7 @@ pub(crate) struct HeaderHints { pub(crate) contiguous_length: u64, } -impl CompactEncodable for HeaderHints { +impl CompactEncoding for HeaderHints { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!(self, reorgs, contiguous_length)) } @@ -304,26 +219,7 @@ impl CompactEncodable for HeaderHints { } } -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderHints) -> Result { - self.preencode(&value.reorgs)?; - self.preencode(&value.contiguous_length) - } - - fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) -> Result { - self.encode(&value.reorgs, buffer)?; - self.encode(&value.contiguous_length, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - Ok(HeaderHints { - reorgs: self.decode(buffer)?, - contiguous_length: self.decode(buffer)?, - }) - } -} - -impl CompactEncodable for Header { +impl CompactEncoding for Header { fn encoded_size(&self) -> Result { Ok(1 + 1 + 32 + sum_encoded_size!(self, manifest, key_pair, user_data, tree, hints)) } @@ -364,63 +260,6 @@ impl CompactEncodable for Header { } } -impl CompactEncoding
for State { - fn preencode(&mut self, value: &Header) -> Result { - self.add_end(1)?; // Version - self.add_end(1)?; // Flags - self.preencode_fixed_32()?; // key - self.preencode(&value.manifest)?; - self.preencode(&value.key_pair)?; - self.preencode(&value.user_data)?; - self.preencode(&value.tree)?; - self.preencode(&value.hints) - } - - fn encode(&mut self, value: &Header, buffer: &mut [u8]) -> Result { - self.set_byte_to_buffer(1, buffer)?; // Version - let flags: u8 = 2 | 4; // Manifest and key pair, TODO: external=1 - self.set_byte_to_buffer(flags, buffer)?; - self.encode_fixed_32(&value.key, buffer)?; - self.encode(&value.manifest, buffer)?; - self.encode(&value.key_pair, buffer)?; - self.encode(&value.user_data, buffer)?; - self.encode(&value.tree, buffer)?; - self.encode(&value.hints, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let version: u8 = self.decode_u8(buffer)?; - if version != 1 { - panic!("Unknown oplog version {}", version); - } - let _flags: u8 = self.decode_u8(buffer)?; - let key: [u8; 32] = self - .decode_fixed_32(buffer)? - .to_vec() - .try_into() - .map_err(|_err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - "Invalid key in oplog header", - ) - })?; - let manifest: Manifest = self.decode(buffer)?; - let key_pair: PartialKeypair = self.decode(buffer)?; - let user_data: Vec = self.decode(buffer)?; - let tree: HeaderTree = self.decode(buffer)?; - let hints: HeaderHints = self.decode(buffer)?; - - Ok(Header { - key, - manifest, - key_pair, - user_data, - tree, - hints, - }) - } -} - #[cfg(test)] mod tests { use super::*; From 0889fd8aba2956c3583bd5fb1f667b2e546def8a Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 19 Apr 2025 17:20:37 -0400 Subject: [PATCH 11/32] lint --- src/crypto/hash.rs | 4 +--- src/oplog/entry.rs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index d8f7e34..c62782a 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -3,9 +3,7 @@ use blake2::{ Blake2b, Blake2bMac, Digest, }; use byteorder::{BigEndian, WriteBytesExt}; -use compact_encoding::{ - as_array, to_encoded_bytes, CompactEncoding, EncodingError, FixedWidthEncoding, -}; +use compact_encoding::{as_array, to_encoded_bytes, EncodingError, FixedWidthEncoding}; use ed25519_dalek::VerifyingKey; use merkle_tree_stream::Node as NodeTrait; use std::convert::AsRef; diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 83571de..455f055 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -62,7 +62,7 @@ impl CompactEncoding for BitfieldUpdate { /// Oplog Entry #[derive(Debug)] -pub struct Entry { +pub(crate) struct Entry { // TODO: This is a keyValueArray in JS pub(crate) user_data: Vec, pub(crate) tree_nodes: Vec, From 1464853a9b1df2b13ea84f85965898097178ce83 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 19 Apr 2025 17:31:17 -0400 Subject: [PATCH 12/32] Fix bug in decoding partial kp --- src/oplog/header.rs | 91 +++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 56b9a47..db117ec 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -156,6 +156,8 @@ impl CompactEncoding for PartialKeypair { where Self: Sized, { + // the ful secret/private key contains the public key duplicated in it + const FULL_SIGNING_KEY_LENGTH: usize = SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH; let (pk_len, rest) = decode_usize(buffer)?; let (public, rest) = match pk_len { PUBLIC_KEY_LENGTH => { @@ -169,20 +171,23 @@ impl CompactEncoding for PartialKeypair { } len => { return Err(EncodingError::invalid_data(&format!( - "Incorrect secret key length while decoding. length = [{len}]" + "Incorrect public key length while decoding. length = [{len}] expected [{PUBLIC_KEY_LENGTH}]" ))) } }; let (sk_len, rest) = decode_usize(rest)?; let (secret, rest) = match sk_len { 0 => (None, rest), - SECRET_KEY_LENGTH => { - let (sk_bytes, rest) = take_array::(rest)?; + // full signing key = secret_key.cocat(public_key) + FULL_SIGNING_KEY_LENGTH => { + let (full_key_bytes, rest) = take_array::(rest)?; + let (sk_bytes, _shoul_be_empty) = take_array::(&full_key_bytes)?; + assert!(_shoul_be_empty.is_empty()); (Some(SigningKey::from_bytes(&sk_bytes)), rest) } len => { return Err(EncodingError::invalid_data(&format!( - "Incorrect secret key length while decoding. length = [{len}]" + "Incorrect secret key length while decoding. length = [{len}] expected [{FULL_SIGNING_KEY_LENGTH}]" ))) } }; @@ -262,98 +267,86 @@ impl CompactEncoding for Header { #[cfg(test)] mod tests { + use compact_encoding::{map_decode, to_encoded_bytes}; + use super::*; use crate::crypto::generate_signing_key; #[test] fn encode_partial_key_pair() -> Result<(), EncodingError> { - let mut enc_state = State::new(); let signing_key = generate_signing_key(); let key_pair = PartialKeypair { public: signing_key.verifying_key(), secret: Some(signing_key), }; - enc_state.preencode(&key_pair)?; - let mut buffer = enc_state.create_buffer(); - // Pub key: 1 byte for length, 32 bytes for content - // Sec key: 1 byte for length, 64 bytes for data + + // sizeof(pk.len()) + sizeof(pk) + sizeof(sk.len() + sizeof(sk) let expected_len = 1 + 32 + 1 + 64; - assert_eq!(buffer.len(), expected_len); - assert_eq!(enc_state.end(), expected_len); - assert_eq!(enc_state.start(), 0); - enc_state.encode(&key_pair, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let key_pair_ret: PartialKeypair = dec_state.decode(&buffer)?; - assert_eq!(key_pair.public, key_pair_ret.public); + let encoded = to_encoded_bytes!(&key_pair); + assert_eq!(encoded.len(), expected_len); + let ((dec_kp,), rest) = map_decode!(&encoded, [PartialKeypair]); + dbg!(rest); + assert!(rest.is_empty()); + assert_eq!(key_pair.public, dec_kp.public); assert_eq!( key_pair.secret.unwrap().to_bytes(), - key_pair_ret.secret.unwrap().to_bytes() + dec_kp.secret.unwrap().to_bytes() ); Ok(()) } #[test] fn encode_tree() -> Result<(), EncodingError> { - let mut enc_state = State::new(); let tree = HeaderTree::new(); - enc_state.preencode(&tree)?; - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&tree, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let tree_ret: HeaderTree = dec_state.decode(&buffer)?; - assert_eq!(tree, tree_ret); + let encoded = to_encoded_bytes!(tree); + // all sizeof(0) + sizeof(0) + sizeof(vec![]) + sizeof(vec![]) == 4 + assert_eq!(encoded.len(), 4); + let ((dec_tree,), rest) = map_decode!(&encoded, [HeaderTree]); + assert!(rest.is_empty()); + assert_eq!(dec_tree, tree); Ok(()) } #[test] - fn encode_tree_cmp() -> Result<(), EncodingError> { - let mut enc_state = State::new(); + fn encode_tree_with_data() -> Result<(), EncodingError> { let tree = HeaderTree { fork: 520, length: 647, root_hash: vec![12; 464].into_boxed_slice(), signature: vec![46; 22].into_boxed_slice(), }; - enc_state.preencode(&tree)?; - //let mut buffer = enc_state.create_buffer(); - let mut buffer = vec![0; enc_state.end()]; - enc_state.encode(&tree, &mut buffer)?; - let mut buf2 = vec![0; tree.encoded_size()?]; - assert_eq!(buffer.len(), buf2.len()); - tree.encode(&mut buf2)?; - assert_eq!(buffer, buf2); - - //assert_eq!(tree, tree_ret); + let encoded = to_encoded_bytes!(&tree); + let ((dec_tree,), rest) = map_decode!(&encoded, [HeaderTree]); + assert!(rest.is_empty()); + assert_eq!(dec_tree, tree); Ok(()) } #[test] fn encode_header() -> Result<(), EncodingError> { - let mut enc_state = State::new(); + //let mut enc_state = State::new(); let signing_key = generate_signing_key(); let signing_key = PartialKeypair { public: signing_key.verifying_key(), secret: Some(signing_key), }; let header = Header::new(signing_key); - enc_state.preencode(&header)?; - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let header_ret: Header = dec_state.decode(&buffer)?; - assert_eq!(header.key_pair.public, header_ret.key_pair.public); - assert_eq!(header.tree.fork, header_ret.tree.fork); - assert_eq!(header.tree.length, header_ret.tree.length); - assert_eq!(header.tree.length, header_ret.tree.length); - assert_eq!(header.manifest.hash, header_ret.manifest.hash); + let encoded = to_encoded_bytes!(&header); + let ((dec_header,), rest) = map_decode!(&encoded, [Header]); + assert!(rest.is_empty()); + assert_eq!(header.key_pair.public, dec_header.key_pair.public); + assert_eq!(header.tree.fork, dec_header.tree.fork); + assert_eq!(header.tree.length, dec_header.tree.length); + assert_eq!(header.tree.length, dec_header.tree.length); + assert_eq!(header.manifest.hash, dec_header.manifest.hash); assert_eq!( header.manifest.signer.public_key, - header_ret.manifest.signer.public_key + dec_header.manifest.signer.public_key ); assert_eq!( header.manifest.signer.signature, - header_ret.manifest.signer.signature + dec_header.manifest.signer.signature ); Ok(()) } From 93f9fabac63e418751ad95e292eee5f95053d1ac Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 19 Apr 2025 17:35:50 -0400 Subject: [PATCH 13/32] BROKEN Remove HypercoreState from oplog/mod.rs --- src/oplog/mod.rs | 233 ++++++++++++++++++++++++----------------------- 1 file changed, 117 insertions(+), 116 deletions(-) diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 6c72020..fc686b5 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,12 +1,15 @@ +use compact_encoding::{ + as_array_mut, get_slices_checked, get_slices_mut_checked, map_decode, take_array_mut, + CompactEncoding, FixedWidthEncoding, FixedWidthU32, +}; use futures::future::Either; use std::convert::{TryFrom, TryInto}; use crate::common::{BitfieldUpdate, Store, StoreInfo, StoreInfoInstruction}; -use crate::encoding::{CompactEncoding, HypercoreState}; use crate::tree::MerkleTreeChangeset; use crate::{HypercoreError, Node, PartialKeypair}; -mod entry; +pub(crate) mod entry; mod header; pub(crate) use entry::{Entry, EntryTreeUpgrade}; @@ -15,6 +18,13 @@ pub(crate) use header::{Header, HeaderTree}; pub(crate) const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; const HEADER_SIZE: usize = 4096; +// NB: we use the word "leader" to describe the 8 byte part put before a chunk of data that +// contains 4 byte checksum, then a 30 bit unsigned integer, then a "header bit" and a "partial +// bit" +const CRC_SIZE: usize = 4; +const LEN_PARTIAL_AND_HEADER_INFO_SIZE: usize = 4; +const LEADER_SIZE: usize = CRC_SIZE + LEN_PARTIAL_AND_HEADER_INFO_SIZE; + /// Oplog. /// /// There are two memory areas for an `Header` in `RandomAccessStorage`: one is the current @@ -73,8 +83,8 @@ enum OplogSlot { } #[derive(Debug)] -struct ValidateLeaderOutcome { - state: HypercoreState, +struct ValidateLeaderOutcome<'a> { + state: &'a [u8], header_bit: bool, partial_bit: bool, } @@ -96,25 +106,27 @@ impl Oplog { Some(info) => { let existing = info.data.expect("Could not get data of existing oplog"); // First read and validate both headers stored in the existing oplog - let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; - let h2_outcome = - Self::validate_leader(OplogSlot::SecondHeader as usize, &existing)?; - + let h1_outcome = Self::validate_leader( + get_slices_checked(&existing, OplogSlot::FirstHeader as usize)?.1, + )?; + let h2_outcome = Self::validate_leader( + get_slices_checked(&existing, OplogSlot::SecondHeader as usize)?.1, + )?; // Depending on what is stored, the state needs to be set accordingly. // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. - let mut outcome: OplogOpenOutcome = if let Some(mut h1_outcome) = h1_outcome { + let mut outcome: OplogOpenOutcome = if let Some(h1_outcome) = h1_outcome { let (header, header_bits): (Header, [bool; 2]) = - if let Some(mut h2_outcome) = h2_outcome { + if let Some(h2_outcome) = h2_outcome { let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; let header: Header = if header_bits[0] == header_bits[1] { - (*h1_outcome.state).decode(&existing)? + Header::decode(h1_outcome.state)?.0 } else { - (*h2_outcome.state).decode(&existing)? + Header::decode(h2_outcome.state)?.0 }; (header, header_bits) } else { ( - (*h1_outcome.state).decode(&existing)?, + Header::decode(h1_outcome.state)?.0, [h1_outcome.header_bit, h1_outcome.header_bit], ) }; @@ -124,7 +136,7 @@ impl Oplog { entries_byte_length: 0, }; OplogOpenOutcome::new(oplog, header, Box::new([])) - } else if let Some(mut h2_outcome) = h2_outcome { + } else if let Some(h2_outcome) = h2_outcome { // This shouldn't happen because the first header is saved to the first slot // but Javascript supports this so we should too. let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; @@ -133,11 +145,7 @@ impl Oplog { entries_length: 0, entries_byte_length: 0, }; - OplogOpenOutcome::new( - oplog, - (*h2_outcome.state).decode(&existing)?, - Box::new([]), - ) + OplogOpenOutcome::new(oplog, Header::decode(h2_outcome.state)?.0, Box::new([])) } else if let Some(key_pair) = key_pair { // There is nothing in the oplog, start from fresh given key pair. Self::fresh(key_pair.clone())? @@ -150,16 +158,15 @@ impl Oplog { // Read headers that might be stored in the existing content if existing.len() > OplogSlot::Entries as usize { - let mut entry_offset = OplogSlot::Entries as usize; + let mut entries_buff = + get_slices_checked(&existing, OplogSlot::Entries as usize)?.1; let mut entries: Vec = Vec::new(); let mut partials: Vec = Vec::new(); - while let Some(mut entry_outcome) = - Self::validate_leader(entry_offset, &existing)? - { - let entry: Entry = entry_outcome.state.decode(&existing)?; - entries.push(entry); + while let Some(entry_outcome) = Self::validate_leader(entries_buff)? { + let res = Entry::decode(entry_outcome.state)?; + entries.push(res.0); + entries_buff = res.1; partials.push(entry_outcome.partial_bit); - entry_offset = (*entry_outcome.state).end(); } // Remove all trailing partial entries @@ -264,7 +271,7 @@ impl Oplog { let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits, clear_traces)?; let mut combined_infos_to_flush: Vec = - infos_to_flush.into_vec().drain(0..1).into_iter().collect(); + infos_to_flush.into_vec().drain(0..1).collect(); let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, new_header_bits, clear_traces)?; combined_infos_to_flush.extend(infos_to_flush.into_vec()); @@ -286,31 +293,25 @@ impl Oplog { ) -> Result, HypercoreError> { let len = batch.len(); let header_bit = self.get_current_header_bit(); - // Leave room for leaders - let mut state = HypercoreState::new_with_start_and_end(0, len * 8); - for entry in batch.iter() { - state.preencode(entry)?; + let mut size = len * LEADER_SIZE; + + // TODO: should I add back the fn sum_encoded_size(&[impl CompactEncoding])-> usize? + // it could be used here. I thought there would not be a case where we were encoding a + // runtime defined number of types in a row and it not be as a Vec (length prefixed) thing + for e in batch.iter() { + size += e.encoded_size()?; } - let mut buffer = state.create_buffer(); + let mut buffer = vec![0; size]; + let mut rest = buffer.as_mut_slice(); for (i, entry) in batch.iter().enumerate() { - (*state).add_start(8)?; - let start = state.start(); let partial_bit: bool = atomic && i < len - 1; - state.encode(entry, &mut buffer)?; - Self::prepend_leader( - state.start() - start, - header_bit, - partial_bit, - &mut state, - &mut buffer, - )?; + rest = encode_with_leader(entry, partial_bit, header_bit, rest)?; } - let index = OplogSlot::Entries as u64 + self.entries_byte_length; self.entries_length += len as u64; - self.entries_byte_length += buffer.len() as u64; + self.entries_byte_length += size as u64; Ok(vec![StoreInfo::new_content(Store::Oplog, index, &buffer)].into_boxed_slice()) } @@ -342,8 +343,6 @@ impl Oplog { clear_traces: bool, ) -> Result<([bool; 2], Box<[StoreInfo]>), HypercoreError> { // The first 8 bytes will be filled with `prepend_leader`. - let data_start_index: usize = 8; - let mut state = HypercoreState::new_with_start_and_end(data_start_index, data_start_index); // Get the right slot and header bit let (oplog_slot, header_bit) = @@ -357,32 +356,17 @@ impl Oplog { } } - // Preencode the new header - (*state).preencode(header)?; + let mut size = LEADER_SIZE + header.encoded_size()?; + size += header.encoded_size()?; // If clearing, lets add zeros to the end - let end = if clear_traces { - let end = state.end(); - state.set_end(HEADER_SIZE); - end - } else { - state.end() - }; + if clear_traces { + size = HEADER_SIZE; + } // Create a buffer for the needed data - let mut buffer = state.create_buffer(); - - // Encode the header - (*state).encode(header, &mut buffer)?; - - // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits - Self::prepend_leader( - end - data_start_index, - header_bit, - false, - &mut state, - &mut buffer, - )?; + let mut buffer = vec![0; size]; + encode_with_leader(header, false, header_bit, &mut buffer)?; // The oplog is always truncated to the minimum byte size, which is right after // all of the entries in the oplog finish. @@ -397,76 +381,39 @@ impl Oplog { )) } - /// Prepends given `State` with 4 bytes of CRC followed by 4 bytes containing length of - /// following buffer, 1 bit indicating which header is relevant to the entry (or if used to - /// wrap the actual header, then the header bit relevant for saving) and 1 bit that tells if - /// the written batch is only partially finished. For this to work, the state given must have - /// 8 bytes in reserve in the beginning, so that state.start can be set back 8 bytes. - fn prepend_leader( - len: usize, - header_bit: bool, - partial_bit: bool, - state: &mut HypercoreState, - buffer: &mut Box<[u8]>, - ) -> Result<(), HypercoreError> { - // The 4 bytes right before start of data is the length in 8+8+8+6=30 bits. The 31st bit is - // the partial bit and 32nd bit the header bit. - let start = (*state).start(); - (*state).set_start(start - len - 4)?; - let len_u32: u32 = len.try_into().unwrap(); - let partial_bit: u32 = if partial_bit { 2 } else { 0 }; - let header_bit: u32 = if header_bit { 1 } else { 0 }; - let combined: u32 = (len_u32 << 2) | header_bit | partial_bit; - state.encode_u32(combined, buffer)?; - - // Before that, is a 4 byte CRC32 that is a checksum of the above encoded 4 bytes and the - // content. - let start = state.start(); - state.set_start(start - 8)?; - let checksum = crc32fast::hash(&buffer[state.start() + 4..state.start() + 8 + len]); - state.encode_u32(checksum, buffer)?; - Ok(()) - } - /// Validates that leader at given index is valid, and returns header and partial bits and /// `State` for the header/entry that the leader was for. - fn validate_leader( - index: usize, - buffer: &[u8], - ) -> Result, HypercoreError> { - if buffer.len() < index + 8 { + fn validate_leader(buffer: &[u8]) -> Result>, HypercoreError> { + if buffer.len() < 8 { return Ok(None); } - let mut state = HypercoreState::new_with_start_and_end(index, buffer.len()); - let stored_checksum: u32 = state.decode_u32(buffer)?; - let combined: u32 = state.decode_u32(buffer)?; + let ((stored_checksum, combined), data_buff) = + map_decode!(buffer, [FixedWidthU32<'_>, FixedWidthU32<'_>]); + let len = usize::try_from(combined >> 2) .expect("Attempted converting to a 32 bit usize on below 32 bit system"); // NB: In the Javascript version IIUC zero length is caught only with a mismatch // of checksums, which is silently interpreted to only mean "no value". That doesn't sound good: // better to throw an error on mismatch and let the caller at least log the problem. - if len == 0 || state.end() - state.start() < len { + if len == 0 || data_buff.len() < len { return Ok(None); } + let header_bit = combined & 1 == 1; let partial_bit = combined & 2 == 2; - let new_start = index + 8; - state.set_end(new_start + len); - state.set_start(new_start)?; - - let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end()]); + let to_hash = &buffer[CRC_SIZE..LEADER_SIZE + len]; + let calculated_checksum = crc32fast::hash(to_hash); if calculated_checksum != stored_checksum { return Err(HypercoreError::InvalidChecksum { - context: "Calculated signature does not match oplog signature".to_string(), + context: format!("Calculated signature [{calculated_checksum}] does not match oplog signature [{stored_checksum}]"), }); }; - Ok(Some(ValidateLeaderOutcome { header_bit, partial_bit, - state, + state: data_buff, })) } @@ -493,3 +440,57 @@ impl Oplog { } } } + +/// Create a header. 30 bits are the length plus two bits for "partial" and "header" info +fn build_len_and_info_header(data_length: usize, header_bit: bool, partial_bit: bool) -> u32 { + let data_length: u32 = data_length + .try_into() + .expect("Must be able to convert usize to u32"); + const MASK: u32 = (3u32).rotate_right(2); + + if (MASK & data_length) != 0 { + panic!("Data length would overflow. It does not fit in 30 bits"); + } + let partial_bit: u32 = if partial_bit { 2 } else { 0 }; + let header_bit: u32 = if header_bit { 1 } else { 0 }; + (data_length << 2) | header_bit | partial_bit +} + +fn write_leader_parts( + header_bit: bool, + partial_bit: bool, + crc_zone: &mut [u8; CRC_SIZE], + len_and_meta_zone: &mut [u8; LEN_PARTIAL_AND_HEADER_INFO_SIZE], + data: &[u8], +) -> Result<(), HypercoreError> { + // first we write the length and partial data + let len_and_info = build_len_and_info_header(data.len(), header_bit, partial_bit); + (len_and_info.as_fixed_width()).encode(len_and_meta_zone)?; + // next we hash the new header info along with the data + let mut hasher = crc32fast::Hasher::new(); + hasher.update(len_and_meta_zone); + hasher.update(data); + hasher.finalize().as_fixed_width().encode(crc_zone)?; + Ok(()) +} + +fn encode_with_leader<'a>( + thing: &impl CompactEncoding, + partial_bit: bool, + header_bit: bool, + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], HypercoreError> { + let (leader_bytes, data_and_rest) = take_array_mut::(buffer)?; + let enc_size = thing.encoded_size()?; + let (data_buff, rest) = get_slices_mut_checked(data_and_rest, enc_size)?; + let (crc_zone, len_and_meta_zone) = get_slices_mut_checked(leader_bytes, CRC_SIZE)?; + thing.encode(data_buff)?; + write_leader_parts( + header_bit, + partial_bit, + as_array_mut::(crc_zone)?, + as_array_mut::(len_and_meta_zone)?, + data_buff, + )?; + Ok(rest) +} From 0bbf3297234f5de639eaf9c7453f5bf0c6d906f7 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 19 Apr 2025 17:39:54 -0400 Subject: [PATCH 14/32] fix bugs --- src/crypto/hash.rs | 9 +++++++-- src/oplog/header.rs | 3 +-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index c62782a..04e06cc 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -168,8 +168,13 @@ impl Hash { for node in roots { let node = node.as_ref(); - let buffer = (|| Ok::<_, EncodingError>(to_encoded_bytes!(node.index(), node.len())))() - .expect("Encoding u64 should not fail"); + let buffer = (|| { + Ok::<_, EncodingError>(to_encoded_bytes!( + node.index().as_fixed_width(), + node.len().as_fixed_width() + )) + })() + .expect("Encoding u64 should not fail"); hasher.update(node.hash()); hasher.update(&buffer[..8]); diff --git a/src/oplog/header.rs b/src/oplog/header.rs index db117ec..f0250b5 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -181,8 +181,7 @@ impl CompactEncoding for PartialKeypair { // full signing key = secret_key.cocat(public_key) FULL_SIGNING_KEY_LENGTH => { let (full_key_bytes, rest) = take_array::(rest)?; - let (sk_bytes, _shoul_be_empty) = take_array::(&full_key_bytes)?; - assert!(_shoul_be_empty.is_empty()); + let (sk_bytes, _pk_bytes) = take_array::(&full_key_bytes)?; (Some(SigningKey::from_bytes(&sk_bytes)), rest) } len => { From f5d09b36c1e7bc216c2b1a3eb355e317e0247708 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 19 Apr 2025 18:58:51 -0400 Subject: [PATCH 15/32] fix bug in open --- src/oplog/header.rs | 1 - src/oplog/mod.rs | 17 +++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/oplog/header.rs b/src/oplog/header.rs index f0250b5..0fc9cb2 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -285,7 +285,6 @@ mod tests { let encoded = to_encoded_bytes!(&key_pair); assert_eq!(encoded.len(), expected_len); let ((dec_kp,), rest) = map_decode!(&encoded, [PartialKeypair]); - dbg!(rest); assert!(rest.is_empty()); assert_eq!(key_pair.public, dec_kp.public); assert_eq!( diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index fc686b5..64206d4 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -106,12 +106,17 @@ impl Oplog { Some(info) => { let existing = info.data.expect("Could not get data of existing oplog"); // First read and validate both headers stored in the existing oplog - let h1_outcome = Self::validate_leader( - get_slices_checked(&existing, OplogSlot::FirstHeader as usize)?.1, - )?; - let h2_outcome = Self::validate_leader( - get_slices_checked(&existing, OplogSlot::SecondHeader as usize)?.1, - )?; + let h1_outcome = if let Some(h1) = existing.get(OplogSlot::FirstHeader as usize..) { + Self::validate_leader(h1)? + } else { + None + }; + let h2_outcome = if let Some(h2) = existing.get(OplogSlot::SecondHeader as usize..) + { + Self::validate_leader(h2)? + } else { + None + }; // Depending on what is stored, the state needs to be set accordingly. // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. let mut outcome: OplogOpenOutcome = if let Some(h1_outcome) = h1_outcome { From 1d5226bc74f4ffe069375baaad7751090e169bfc Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sun, 20 Apr 2025 20:10:57 -0400 Subject: [PATCH 16/32] fix bug in open --- src/oplog/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 64206d4..2602b35 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -106,12 +106,15 @@ impl Oplog { Some(info) => { let existing = info.data.expect("Could not get data of existing oplog"); // First read and validate both headers stored in the existing oplog - let h1_outcome = if let Some(h1) = existing.get(OplogSlot::FirstHeader as usize..) { + let h1_outcome = if let Some(h1) = + existing.get(OplogSlot::FirstHeader as usize..OplogSlot::SecondHeader as usize) + { Self::validate_leader(h1)? } else { None }; - let h2_outcome = if let Some(h2) = existing.get(OplogSlot::SecondHeader as usize..) + let h2_outcome = if let Some(h2) = + existing.get(OplogSlot::SecondHeader as usize..OplogSlot::Entries as usize) { Self::validate_leader(h2)? } else { From 93c0c06bc14fb97f4ac5be11a0e770835571a604 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sun, 20 Apr 2025 20:11:15 -0400 Subject: [PATCH 17/32] dedupe macros --- src/encoding.rs | 83 ++++++++++++++------------------------------- src/oplog/entry.rs | 26 ++++++++++---- src/oplog/header.rs | 39 ++++++++++++++------- 3 files changed, 71 insertions(+), 77 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index fb67b6f..484b5b8 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -3,50 +3,20 @@ use crate::{ crypto::{Manifest, ManifestSigner}, DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, }; -use compact_encoding::encoded_size_usize; +use compact_encoding::{as_array, encoded_size_usize, map_encode, sum_encoded_size}; pub use compact_encoding::{ bytes_fixed_from_vec, encode_bytes_fixed, take_array, take_array_mut, write_array, write_slice, CompactEncoding, EncodingError, EncodingErrorKind, VecEncodable, }; -#[macro_export] -/// Used for defining CompactEncoding::encoded_size. -/// Pass self and a list of fields to call encoded_size on -macro_rules! sum_encoded_size { - // Base case: single field - ($self:ident, $field:ident) => { - $self.$field.encoded_size()? - }; - // Recursive case: first field + rest - ($self: ident, $first:ident, $($rest:ident),+) => { - $self.$first.encoded_size()? + sum_encoded_size!($self, $($rest),+) - }; -} - -#[macro_export] -// TODO is this exported from the crate? -/// Used for defining CompactEncoding::encode. -/// Pass self, the buffer and a list of fields to call encoded_size on -macro_rules! chain_encoded_bytes { - // Base case: single field - ($self:ident, $buffer:ident, $field:ident) => { - $self.$field.encode($buffer)? - }; - // Recursive case: first field + rest - ($self: ident, $buffer:ident, $first:ident, $($rest:ident),+) => {{ - let rest = $self.$first.encode($buffer)?; - chain_encoded_bytes!($self, rest, $($rest),+) - }}; -} - impl CompactEncoding for Node { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, index, length) + 32) + Ok(sum_encoded_size!(self.index, self.length) + 32) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let rest = chain_encoded_bytes!(self, buffer, index, length); - bytes_fixed_from_vec::<32>(&self.hash)?.encode(rest) + let hash = as_array::<32>(&self.hash)?; + Ok(map_encode!(buffer, self.index, self.length, hash)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -56,6 +26,7 @@ impl CompactEncoding for Node { let (index, rest) = u64::decode(buffer)?; let (length, rest) = u64::decode(rest)?; let (hash, rest) = <[u8; 32]>::decode(rest)?; + Ok((Node::new(index, hash.to_vec(), length), rest)) } } @@ -75,11 +46,11 @@ impl VecEncodable for Node { impl CompactEncoding for RequestBlock { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, index, nodes)) + Ok(sum_encoded_size!(self.index, self.nodes)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!(self, buffer, index, nodes)) + Ok(map_encode!(buffer, self.index, self.nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -112,11 +83,11 @@ impl CompactEncoding for RequestSeek { impl CompactEncoding for RequestUpgrade { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, start, length)) + Ok(sum_encoded_size!(self.start, self.length)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!(self, buffer, start, length)) + Ok(map_encode!(buffer, self.start, self.length)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -131,11 +102,11 @@ impl CompactEncoding for RequestUpgrade { impl CompactEncoding for DataBlock { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, index, value, nodes)) + Ok(sum_encoded_size!(self.index, self.value, self.nodes)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!(self, buffer, index, value, nodes)) + Ok(map_encode!(buffer, self.index, self.value, self.nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -158,11 +129,11 @@ impl CompactEncoding for DataBlock { impl CompactEncoding for DataHash { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, index, nodes)) + Ok(sum_encoded_size!(self.index, self.nodes)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!(self, buffer, index, nodes)) + Ok(map_encode!(buffer, self.index, self.nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -177,11 +148,11 @@ impl CompactEncoding for DataHash { impl CompactEncoding for DataSeek { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, bytes, nodes)) + Ok(sum_encoded_size!(self.bytes, self.nodes)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!(self, buffer, bytes, nodes)) + Ok(map_encode!(buffer, self.bytes, self.nodes)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -197,24 +168,22 @@ impl CompactEncoding for DataSeek { impl CompactEncoding for DataUpgrade { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!( - self, - start, - length, - nodes, - additional_nodes, - signature + self.start, + self.length, + self.nodes, + self.additional_nodes, + self.signature )) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!( - self, + Ok(map_encode!( buffer, - start, - length, - nodes, - additional_nodes, - signature + self.start, + self.length, + self.nodes, + self.additional_nodes, + self.signature )) } diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 455f055..764b019 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,6 +1,9 @@ -use compact_encoding::{take_array, take_array_mut, write_array, CompactEncoding, EncodingError}; +use compact_encoding::{ + map_encode, sum_encoded_size, take_array, take_array_mut, write_array, CompactEncoding, + EncodingError, +}; -use crate::{chain_encoded_bytes, decode, sum_encoded_size}; +use crate::decode; use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade @@ -14,12 +17,21 @@ pub(crate) struct EntryTreeUpgrade { impl CompactEncoding for EntryTreeUpgrade { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, fork, ancestors, length, signature)) + Ok(sum_encoded_size!( + self.fork, + self.ancestors, + self.length, + self.signature + )) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!( - self, buffer, fork, ancestors, length, signature + Ok(map_encode!( + buffer, + self.fork, + self.ancestors, + self.length, + self.signature )) } @@ -33,13 +45,13 @@ impl CompactEncoding for EntryTreeUpgrade { impl CompactEncoding for BitfieldUpdate { fn encoded_size(&self) -> Result { - Ok(1 + sum_encoded_size!(self, start, length)) + Ok(1 + sum_encoded_size!(self.start, self.length)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let drop = if self.drop { 1 } else { 0 }; let rest = write_array(&[drop], buffer)?; - Ok(chain_encoded_bytes!(self, rest, start, length)) + Ok(map_encode!(rest, self.start, self.length)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 0fc9cb2..30f974c 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,10 +1,11 @@ use compact_encoding::{decode_usize, take_array, write_array, CompactEncoding, EncodingError}; +use compact_encoding::{map_encode, sum_encoded_size}; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use crate::crypto::default_signer_manifest; use crate::crypto::Manifest; -use crate::{chain_encoded_bytes, VerifyingKey}; -use crate::{sum_encoded_size, PartialKeypair}; +use crate::PartialKeypair; +use crate::VerifyingKey; /// Oplog header. #[derive(Debug, Clone)] @@ -110,12 +111,21 @@ macro_rules! decode { impl CompactEncoding for HeaderTree { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, fork, length, root_hash, signature)) + Ok(sum_encoded_size!( + self.fork, + self.length, + self.root_hash, + self.signature + )) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!( - self, buffer, fork, length, root_hash, signature + Ok(map_encode!( + buffer, + self.fork, + self.length, + self.root_hash, + self.signature )) } @@ -203,16 +213,11 @@ pub(crate) struct HeaderHints { impl CompactEncoding for HeaderHints { fn encoded_size(&self) -> Result { - Ok(sum_encoded_size!(self, reorgs, contiguous_length)) + Ok(sum_encoded_size!(self.reorgs, self.contiguous_length)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - Ok(chain_encoded_bytes!( - self, - buffer, - reorgs, - contiguous_length - )) + Ok(map_encode!(buffer, self.reorgs, self.contiguous_length)) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -225,7 +230,15 @@ impl CompactEncoding for HeaderHints { impl CompactEncoding for Header { fn encoded_size(&self) -> Result { - Ok(1 + 1 + 32 + sum_encoded_size!(self, manifest, key_pair, user_data, tree, hints)) + Ok(1 + 1 + + 32 + + sum_encoded_size!( + self.manifest, + self.key_pair, + self.user_data, + self.tree, + self.hints + )) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { From c9c1ca597410fe6500c69e54b7c2f1c3ba02b947 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sun, 20 Apr 2025 20:12:52 -0400 Subject: [PATCH 18/32] fix decode bug --- src/oplog/entry.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 764b019..8e2f44f 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -135,20 +135,20 @@ impl CompactEncoding for Entry { }; let (tree_nodes, rest) = if flags & 2 != 0 { - >::decode(buffer)? + >::decode(rest)? } else { (Default::default(), rest) }; let (tree_upgrade, rest) = if flags & 2 != 0 { - let (x, rest) = EntryTreeUpgrade::decode(buffer)?; + let (x, rest) = EntryTreeUpgrade::decode(rest)?; (Some(x), rest) } else { (Default::default(), rest) }; let (bitfield, rest) = if flags & 2 != 0 { - let (x, rest) = BitfieldUpdate::decode(buffer)?; + let (x, rest) = BitfieldUpdate::decode(rest)?; (Some(x), rest) } else { (Default::default(), rest) From f741fc1351dbb61a9fe19d660da90c4565d258aa Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 12:36:30 -0400 Subject: [PATCH 19/32] export more funcs used in protocol --- src/encoding.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index 484b5b8..980c8f3 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -3,9 +3,10 @@ use crate::{ crypto::{Manifest, ManifestSigner}, DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, }; -use compact_encoding::{as_array, encoded_size_usize, map_encode, sum_encoded_size}; +use compact_encoding::{as_array, encoded_size_usize}; pub use compact_encoding::{ - bytes_fixed_from_vec, encode_bytes_fixed, take_array, take_array_mut, write_array, write_slice, + bytes_fixed_from_vec, decode_usize, encode_bytes_fixed, map_decode, map_encode, + sum_encoded_size, take_array, take_array_mut, to_encoded_bytes, write_array, write_slice, CompactEncoding, EncodingError, EncodingErrorKind, VecEncodable, }; From 819b943803aa448853f6efc161bf979c9d450622 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 12:37:00 -0400 Subject: [PATCH 20/32] fix bug encoding DataUpgrade --- src/encoding.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index 980c8f3..db0323a 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -166,6 +166,8 @@ impl CompactEncoding for DataSeek { } } +// from: +// https://github.com/holepunchto/hypercore/blob/d21ebdeca1b27eb4c2232f8af17d5ae939ee97f2/lib/messages.js#L394 impl CompactEncoding for DataUpgrade { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!( @@ -196,14 +198,14 @@ impl CompactEncoding for DataUpgrade { let (length, rest) = u64::decode(rest)?; let (nodes, rest) = Vec::::decode(rest)?; let (additional_nodes, rest) = Vec::::decode(rest)?; - let (signature, rest) = <[u8; 32]>::decode(rest)?; + let (signature, rest) = >::decode(rest)?; Ok(( DataUpgrade { start, length, nodes, additional_nodes, - signature: signature.to_vec(), + signature, }, rest, )) From 3069b600a1af7483e9b1d4b7a48426dc70d80ea4 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 17:16:23 -0400 Subject: [PATCH 21/32] CompactEncoding Vec -> Box<[u8]> --- src/crypto/hash.rs | 5 +++-- src/tree/merkle_tree.rs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 04e06cc..fcf0400 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -147,8 +147,9 @@ impl Hash { }; let len = node1.length + node2.length; - let size: Vec = (|| Ok::<_, EncodingError>(to_encoded_bytes!(len.as_fixed_width())))() - .expect("Encoding u64 should not fail"); + let size: Box<[u8]> = + (|| Ok::<_, EncodingError>(to_encoded_bytes!(len.as_fixed_width())))() + .expect("Encoding u64 should not fail"); let mut hasher = Blake2b256::new(); hasher.update(PARENT_TYPE); diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index e7bca4d..32f7f0a 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -662,7 +662,10 @@ impl MerkleTree { for (_, node) in self.unflushed.drain() { let buffer = (|| { let hash = as_array::<32>(&node.hash)?; - Ok::, EncodingError>(to_encoded_bytes!(node.length.as_fixed_width(), hash)) + Ok::, EncodingError>(to_encoded_bytes!( + node.length.as_fixed_width(), + hash + )) })() .expect("Encoding u64 should not fail"); infos_to_flush.push(StoreInfo::new_content( From 2788bb109f9680f4feaa55e1da7392eb57360394 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 18:59:05 -0400 Subject: [PATCH 22/32] Don't re-export compact encoding --- src/encoding.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index db0323a..1f13063 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -3,11 +3,9 @@ use crate::{ crypto::{Manifest, ManifestSigner}, DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, }; -use compact_encoding::{as_array, encoded_size_usize}; -pub use compact_encoding::{ - bytes_fixed_from_vec, decode_usize, encode_bytes_fixed, map_decode, map_encode, - sum_encoded_size, take_array, take_array_mut, to_encoded_bytes, write_array, write_slice, - CompactEncoding, EncodingError, EncodingErrorKind, VecEncodable, +use compact_encoding::{ + as_array, encode_bytes_fixed, encoded_size_usize, map_decode, map_encode, sum_encoded_size, + take_array, write_slice, CompactEncoding, EncodingError, EncodingErrorKind, VecEncodable, }; impl CompactEncoding for Node { From 804631a9d0850e5cb52c9e8496055e0a1fae52ac Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 18:59:49 -0400 Subject: [PATCH 23/32] use map_decode --- src/encoding.rs | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/encoding.rs b/src/encoding.rs index 1f13063..b54fd05 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -22,10 +22,7 @@ impl CompactEncoding for Node { where Self: Sized, { - let (index, rest) = u64::decode(buffer)?; - let (length, rest) = u64::decode(rest)?; - let (hash, rest) = <[u8; 32]>::decode(rest)?; - + let ((index, length, hash), rest) = map_decode!(buffer, [u64, u64, [u8; 32]]); Ok((Node::new(index, hash.to_vec(), length), rest)) } } @@ -56,8 +53,7 @@ impl CompactEncoding for RequestBlock { where Self: Sized, { - let (index, rest) = u64::decode(buffer)?; - let (nodes, rest) = u64::decode(rest)?; + let ((index, nodes), rest) = map_decode!(buffer, [u64, u64]); Ok((RequestBlock { index, nodes }, rest)) } } @@ -93,8 +89,7 @@ impl CompactEncoding for RequestUpgrade { where Self: Sized, { - let (start, rest) = u64::decode(buffer)?; - let (length, rest) = u64::decode(rest)?; + let ((start, length), rest) = map_decode!(buffer, [u64, u64]); Ok((RequestUpgrade { start, length }, rest)) } } @@ -112,9 +107,7 @@ impl CompactEncoding for DataBlock { where Self: Sized, { - let (index, rest) = u64::decode(buffer)?; - let (value, rest) = Vec::::decode(rest)?; - let (nodes, rest) = Vec::::decode(rest)?; + let ((index, value, nodes), rest) = map_decode!(buffer, [u64, Vec, Vec]); Ok(( DataBlock { index, @@ -139,8 +132,7 @@ impl CompactEncoding for DataHash { where Self: Sized, { - let (index, rest) = u64::decode(buffer)?; - let (nodes, rest) = Vec::::decode(rest)?; + let ((index, nodes), rest) = map_decode!(buffer, [u64, Vec]); Ok((DataHash { index, nodes }, rest)) } } @@ -158,8 +150,7 @@ impl CompactEncoding for DataSeek { where Self: Sized, { - let (bytes, rest) = u64::decode(buffer)?; - let (nodes, rest) = Vec::::decode(rest)?; + let ((bytes, nodes), rest) = map_decode!(buffer, [u64, Vec]); Ok((DataSeek { bytes, nodes }, rest)) } } @@ -192,11 +183,8 @@ impl CompactEncoding for DataUpgrade { where Self: Sized, { - let (start, rest) = u64::decode(buffer)?; - let (length, rest) = u64::decode(rest)?; - let (nodes, rest) = Vec::::decode(rest)?; - let (additional_nodes, rest) = Vec::::decode(rest)?; - let (signature, rest) = >::decode(rest)?; + let ((start, length, nodes, additional_nodes, signature), rest) = + map_decode!(buffer, [u64, u64, Vec, Vec, Vec]); Ok(( DataUpgrade { start, From be60bcad5e40d8e723e5987b9629989ea8a1fd10 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 19:08:36 -0400 Subject: [PATCH 24/32] remove decode! macro and use map_decode --- src/oplog/entry.rs | 17 ++++++++++---- src/oplog/header.rs | 54 ++++++++++++++++++--------------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 8e2f44f..35cba4b 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,9 +1,8 @@ use compact_encoding::{ - map_encode, sum_encoded_size, take_array, take_array_mut, write_array, CompactEncoding, - EncodingError, + map_decode, map_encode, sum_encoded_size, take_array, take_array_mut, write_array, + CompactEncoding, EncodingError, }; -use crate::decode; use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade @@ -39,7 +38,17 @@ impl CompactEncoding for EntryTreeUpgrade { where Self: Sized, { - decode!(EntryTreeUpgrade, buffer, {fork: u64, ancestors: u64, length: u64, signature: Box<[u8]>}) + let ((fork, ancestors, length, signature), rest) = + map_decode!(buffer, [u64, u64, u64, Box<[u8]>]); + Ok(( + Self { + fork, + ancestors, + length, + signature, + }, + rest, + )) } } diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 30f974c..88e982e 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,4 +1,6 @@ -use compact_encoding::{decode_usize, take_array, write_array, CompactEncoding, EncodingError}; +use compact_encoding::{ + decode_usize, map_decode, take_array, write_array, CompactEncoding, EncodingError, +}; use compact_encoding::{map_encode, sum_encoded_size}; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; @@ -80,35 +82,6 @@ impl HeaderTree { } } -#[macro_export] -/// Helper for decoding a struct with compact encodable -macro_rules! decode { - // Match the pattern: decode!(StructName, buffer, {field1: type1, field2: type2, ...}) - ($struct_name:ident, $buffer:expr, { - $($field_name:ident : $field_type:ty),* $(,)? - }) => {{ - - // Variable to hold the current buffer state - let mut current_buffer = $buffer; - - // Decode each field in sequence - $( - let ($field_name, new_buffer) = <$field_type>::decode(current_buffer)?; - current_buffer = new_buffer; - )* - - // Create the struct with decoded fields - let result = $struct_name { - $( - $field_name, - )* - }; - - // Return the struct and the remaining buffer - Ok((result, current_buffer)) - }}; - } - impl CompactEncoding for HeaderTree { fn encoded_size(&self) -> Result { Ok(sum_encoded_size!( @@ -133,7 +106,17 @@ impl CompactEncoding for HeaderTree { where Self: Sized, { - decode!(HeaderTree, buffer, {fork: u64, length: u64, root_hash: Box<[u8]>, signature: Box<[u8]>}) + let ((fork, length, root_hash, signature), rest) = + map_decode!(buffer, [u64, u64, Box<[u8]>, Box<[u8]>]); + Ok(( + Self { + fork, + length, + root_hash, + signature, + }, + rest, + )) } } @@ -224,7 +207,14 @@ impl CompactEncoding for HeaderHints { where Self: Sized, { - decode!(HeaderHints, buffer, {reorgs: Vec, contiguous_length: u64 }) + let ((reorgs, contiguous_length), rest) = map_decode!(buffer, [Vec, u64]); + Ok(( + Self { + reorgs, + contiguous_length, + }, + rest, + )) } } From 3b521e587909482c94f368dd9064484bb78fbc84 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 19:13:19 -0400 Subject: [PATCH 25/32] more map_decode --- src/oplog/header.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 88e982e..c7ad6c2 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -233,13 +233,15 @@ impl CompactEncoding for Header { fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let rest = write_array(&[1, 2 | 4], buffer)?; - let rest = self.key.encode(rest)?; - let rest = self.manifest.encode(rest)?; - let rest = self.key_pair.encode(rest)?; - let rest = self.user_data.encode(rest)?; - let rest = self.tree.encode(rest)?; - let rest = self.hints.encode(rest)?; - Ok(rest) + Ok(map_encode!( + rest, + self.key, + self.manifest, + self.key_pair, + self.user_data, + self.tree, + self.hints + )) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -248,11 +250,11 @@ impl CompactEncoding for Header { { let ([_version, _flags], rest) = take_array::<2>(buffer)?; let (key, rest) = take_array::<32>(rest)?; - let (manifest, rest) = Manifest::decode(rest)?; - let (key_pair, rest) = PartialKeypair::decode(rest)?; - let (user_data, rest) = >::decode(rest)?; - let (tree, rest) = HeaderTree::decode(rest)?; - let (hints, rest) = HeaderHints::decode(rest)?; + let ((manifest, key_pair, user_data, tree, hints), rest) = map_decode!( + rest, [ + Manifest, PartialKeypair, Vec, HeaderTree, HeaderHints + ] + ); Ok(( Header { key, From ff89a4bbfce34ac28747813b7c2d2c7e5472b6a1 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 19:18:40 -0400 Subject: [PATCH 26/32] remove & add TODO --- src/common/node.rs | 2 ++ src/oplog/mod.rs | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/common/node.rs b/src/common/node.rs index 1c78144..b152779 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -23,6 +23,8 @@ pub struct Node { /// This node's index in the Merkle tree pub(crate) index: u64, /// Hash of the data in this node + // TODO make this [u8; 32] like: + // https://github.com/holepunchto/hypercore/blob/d21ebdeca1b27eb4c2232f8af17d5ae939ee97f2/lib/messages.js#L246 pub(crate) hash: Vec, /// Number of bytes in this [`Node::data`] pub(crate) length: u64, diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 2602b35..eb64cfb 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -304,9 +304,6 @@ impl Oplog { let mut size = len * LEADER_SIZE; - // TODO: should I add back the fn sum_encoded_size(&[impl CompactEncoding])-> usize? - // it could be used here. I thought there would not be a case where we were encoding a - // runtime defined number of types in a row and it not be as a Vec (length prefixed) thing for e in batch.iter() { size += e.encoded_size()?; } From 29e041eed5d59b9fa58dd11b925d88cb3fcebe6f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 5 May 2025 12:46:34 -0400 Subject: [PATCH 27/32] use map_decode and check flag correctly --- src/oplog/entry.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 35cba4b..c8eeac4 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -68,11 +68,10 @@ impl CompactEncoding for BitfieldUpdate { Self: Sized, { let ([flags], rest) = take_array::<1>(buffer)?; - let (start, rest) = u64::decode(rest)?; - let (length, rest) = u64::decode(rest)?; + let ((start, length), rest) = map_decode!(rest, [u64, u64]); Ok(( BitfieldUpdate { - drop: flags == 1, + drop: flags & 1 == 1, start, length, }, From 5577d590a6ad0a899a5dd4b90cc866f1e31b0808 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 5 May 2025 12:48:05 -0400 Subject: [PATCH 28/32] wip use local compact-encoding --- Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 61f3db9..af9f8ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ ed25519-dalek = { version = "2", features = ["rand_core"] } getrandom = { version = "0.2", features = ["js"] } thiserror = "1" tracing = "0.1" -compact-encoding = "1" +#compact-encoding = "1" flat-tree = "6" merkle-tree-stream = "0.12" pretty-hash = "0.4" @@ -45,6 +45,9 @@ async-lock = {version = "3.4.0", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] random-access-disk = { version = "3", default-features = false } +[dependencies.compact-encoding] +path = "../compact-encoding" + [dev-dependencies] anyhow = "1.0.70" proptest = "1.6.0" From 50409793ca0386dc79b3a16f43aedc0f67c0f975 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 5 May 2025 14:10:44 -0400 Subject: [PATCH 29/32] use new release of compact-encoding --- Cargo.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index af9f8ea..a47ed08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ ed25519-dalek = { version = "2", features = ["rand_core"] } getrandom = { version = "0.2", features = ["js"] } thiserror = "1" tracing = "0.1" -#compact-encoding = "1" +compact-encoding = "2" flat-tree = "6" merkle-tree-stream = "0.12" pretty-hash = "0.4" @@ -45,9 +45,6 @@ async-lock = {version = "3.4.0", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] random-access-disk = { version = "3", default-features = false } -[dependencies.compact-encoding] -path = "../compact-encoding" - [dev-dependencies] anyhow = "1.0.70" proptest = "1.6.0" From e571f9844dfbc1c01e80944bd8007cfd85af4142 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 5 May 2025 14:22:03 -0400 Subject: [PATCH 30/32] Make BitfieldUpdate pub(crate) I'm not sure why this was not an error before --- src/common/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/mod.rs b/src/common/mod.rs index f5fb6ba..9df6430 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -16,7 +16,7 @@ pub use self::store::Store; pub(crate) use self::store::{StoreInfo, StoreInfoInstruction, StoreInfoType}; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct BitfieldUpdate { +pub(crate) struct BitfieldUpdate { pub(crate) drop: bool, pub(crate) start: u64, pub(crate) length: u64, From ee5d7befc1ff050dceeffe51d61cc36e60d3d382 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 5 May 2025 14:37:59 -0400 Subject: [PATCH 31/32] clippy fixes --- src/crypto/hash.rs | 1 - src/tree/merkle_tree.rs | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index fcf0400..2653361 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -220,7 +220,6 @@ pub(crate) fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { )) })() .expect("Encoding should not fail") - .into() } #[cfg(test)] diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 32f7f0a..87cb22b 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -94,7 +94,7 @@ impl MerkleTree { if length > 0 { length /= 2; } - let signature: Option = if header_tree.signature.len() > 0 { + let signature: Option = if !header_tree.signature.is_empty() { Some( Signature::try_from(&*header_tree.signature).map_err(|_err| { HypercoreError::InvalidSignature { @@ -481,11 +481,7 @@ impl MerkleTree { start: upgrade.start, length: upgrade.length, nodes: p.upgrade.expect("nodes need to be set"), - additional_nodes: if let Some(additional_upgrade) = p.additional_upgrade { - additional_upgrade - } else { - vec![] - }, + additional_nodes: p.additional_upgrade.unwrap_or_default(), signature: signature .expect("signature needs to be set") .to_bytes() @@ -1566,7 +1562,7 @@ fn parent_node(index: u64, left: &Node, right: &Node) -> Node { ) } -fn block_node(index: u64, value: &Vec) -> Node { +fn block_node(index: u64, value: &[u8]) -> Node { Node::new( index, Hash::data(value).as_bytes().to_vec(), From 860eeb574bb47c5f198ad07e1cc050fe0f0015b7 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 5 May 2025 14:41:06 -0400 Subject: [PATCH 32/32] CI: clippy, check docs and fail on warnings --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0fac45..5d6bcbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ on: env: RUST_BACKTRACE: 1 CARGO_TERM_COLOR: always + RUSTFLAGS: "-Dwarnings" + RUSTDOCFLAGS: "-Dwarnings" jobs: ci-pass: @@ -145,4 +147,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Format check run: | + cargo doc cargo fmt -- --check + cargo clippy --all-targets