From 7910db6e723de6de29703f5763f80e8e186acf6c Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Thu, 9 Mar 2023 13:28:02 -0800 Subject: [PATCH 1/4] Verify round-trip in messagebuf_fromiterator test --- src/message.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/message.rs b/src/message.rs index da49ead..3f1b431 100644 --- a/src/message.rs +++ b/src/message.rs @@ -632,6 +632,8 @@ mod tests { pub fn messagebuf_fromiterator() { let buf = vec!["test", "123"].iter().collect::(); assert_eq!(2, buf.len()); + assert_eq!(b"test", buf.iter().nth(0).unwrap().payload()); + assert_eq!(b"123", buf.iter().nth(1).unwrap().payload()); } #[test] From bba59649a203abe9d4382bfa279c1e6676f53334 Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Thu, 9 Mar 2023 14:47:47 -0800 Subject: [PATCH 2/4] Verify error code is MessageSizeExceeded in message size limit test --- src/lib.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0c708d9..3b604fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -759,19 +759,12 @@ mod tests { pub fn append_message_greater_than_max() { let dir = TestDir::new(); let mut log = CommitLog::new(LogOptions::new(&dir)).unwrap(); - //create vector with 1.2mb of size, u8 = 1 byte thus, - //1mb = 1000000 bytes, 1200000 items needed - let mut value = String::new(); - let mut target = 0; - while target != 2000000 { - value.push('a'); - target += 1; - } + // Try to write a value larger than the 1mb default message_max_bytes limit + let value = std::iter::repeat('a').take(1_200_000).collect::(); let res = log.append_msg(value); - //will fail if no error is found which means a message greater than the limit - // passed through - assert!(res.is_err()); + assert!(matches!(res.err(), Some(AppendError::MessageSizeExceeded))); log.flush().unwrap(); + assert_eq!(0, log.next_offset()); } #[test] From 475264e9581afb6756619744d1d4748bf7e1adee Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Thu, 9 Mar 2023 15:38:11 -0800 Subject: [PATCH 3/4] Verify the message-too-large read error This is a critical part of the read API. Because the default message size write limit (1mb) is larger than the default read size (8kb), clients using the default limits can write messages they cannot read. Clients needs to be able to distinguish this message-too-big-to-read case from other errors so they can retry the read with a larger ReadLimit. --- src/lib.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 3b604fc..6665389 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -677,6 +677,25 @@ mod tests { } } + #[test] + pub fn read_large_message_error() { + let dir = TestDir::new(); + let mut log = CommitLog::new(LogOptions::new(&dir)).unwrap(); + let value = std::iter::repeat('a').take(900_000).collect::(); + log.append_msg(value).unwrap(); + let res = log.read(0, ReadLimit::default()); + match res { + Err(ReadError::Io(err)) => { + assert_eq!(io::ErrorKind::InvalidInput, err.kind()); + assert_eq!( + "Message exceeded max byte size", + err.get_ref().unwrap().to_string() + ); + } + _ => panic!("Expected ReadError::Io"), + } + } + #[test] pub fn reopen_log() { env_logger::try_init().unwrap_or(()); From cb606df9342955ae18236fb8220d6315cfdc4b1e Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Thu, 9 Mar 2023 15:50:58 -0800 Subject: [PATCH 4/4] Verify large messages can be read with a larger ReadLimit --- src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6665389..8f71cb3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -682,7 +682,7 @@ mod tests { let dir = TestDir::new(); let mut log = CommitLog::new(LogOptions::new(&dir)).unwrap(); let value = std::iter::repeat('a').take(900_000).collect::(); - log.append_msg(value).unwrap(); + log.append_msg(&value).unwrap(); let res = log.read(0, ReadLimit::default()); match res { Err(ReadError::Io(err)) => { @@ -694,6 +694,15 @@ mod tests { } _ => panic!("Expected ReadError::Io"), } + assert_eq!( + value.as_bytes(), + log.read(0, ReadLimit::max_bytes(1_000_000)) + .unwrap() + .iter() + .nth(0) + .unwrap() + .payload() + ); } #[test]