From 224c9abc48317de18212994c0486ddd8d0798e07 Mon Sep 17 00:00:00 2001 From: Sean McArthur Date: Fri, 11 Oct 2019 15:02:05 -0700 Subject: [PATCH] Refactor Bytes to use an internal vtable Bytes is a useful tool for managing multiple slices into the same region of memory, and the other things it used to have been removed to reduce complexity. The exact strategy for managing the multiple references is no longer hard-coded, but instead backing by a customizable vtable. - Removed ability to mutate the underlying memory from the `Bytes` type. - Removed the "inline" (SBO) mechanism in `Bytes`. The reduces a large amount of complexity, and improves performance when accessing the slice of bytes, since a branch is no longer needed to check if the data is inline. - Removed `Bytes` knowledge of `BytesMut` (`BytesMut` may grow that knowledge back at a future point.) --- Cargo.toml | 1 + benches/bytes.rs | 172 +-- benches/bytes_mut.rs | 142 ++ ci/azure-cross-compile.yml | 2 +- ci/azure-tsan.yml | 4 +- ci/tsan | 4 - src/buf/chain.rs | 8 - src/buf/vec_deque.rs | 19 - src/bytes.rs | 2838 +++++------------------------------- src/bytes_mut.rs | 1429 ++++++++++++++++++ src/lib.rs | 27 +- src/loom.rs | 5 + src/serde.rs | 14 +- tests/fuzz_bytes.rs | 73 + tests/test_buf.rs | 16 + tests/test_buf_mut.rs | 10 +- tests/test_bytes.rs | 241 +-- 17 files changed, 2176 insertions(+), 2829 deletions(-) create mode 100644 benches/bytes_mut.rs create mode 100644 src/bytes_mut.rs create mode 100644 src/loom.rs create mode 100644 tests/fuzz_bytes.rs diff --git a/Cargo.toml b/Cargo.toml index 746a193fb..bc4d670cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,4 +28,5 @@ serde = { version = "1.0", optional = true } either = { version = "1.5", default-features = false, optional = true } [dev-dependencies] +loom = "0.2.10" serde_test = "1.0" diff --git a/benches/bytes.rs b/benches/bytes.rs index 5885a1bfa..9c36e6081 100644 --- a/benches/bytes.rs +++ b/benches/bytes.rs @@ -4,47 +4,11 @@ extern crate test; use test::Bencher; -use bytes::{Bytes, BytesMut, BufMut}; - -#[bench] -fn alloc_small(b: &mut Bencher) { - b.iter(|| { - for _ in 0..1024 { - test::black_box(BytesMut::with_capacity(12)); - } - }) -} - -#[bench] -fn alloc_mid(b: &mut Bencher) { - b.iter(|| { - test::black_box(BytesMut::with_capacity(128)); - }) -} - -#[bench] -fn alloc_big(b: &mut Bencher) { - b.iter(|| { - test::black_box(BytesMut::with_capacity(4096)); - }) -} - -#[bench] -fn split_off_and_drop(b: &mut Bencher) { - b.iter(|| { - for _ in 0..1024 { - let v = vec![10; 200]; - let mut b = Bytes::from(v); - test::black_box(b.split_off(100)); - test::black_box(b); - } - }) -} +use bytes::Bytes; #[bench] fn deref_unique(b: &mut Bencher) { - let mut buf = BytesMut::with_capacity(4096); - buf.put(&[0u8; 1024][..]); + let buf = Bytes::from(vec![0; 1024]); b.iter(|| { for _ in 0..1024 { @@ -53,30 +17,10 @@ fn deref_unique(b: &mut Bencher) { }) } -#[bench] -fn deref_unique_unroll(b: &mut Bencher) { - let mut buf = BytesMut::with_capacity(4096); - buf.put(&[0u8; 1024][..]); - - b.iter(|| { - for _ in 0..128 { - test::black_box(&buf[..]); - test::black_box(&buf[..]); - test::black_box(&buf[..]); - test::black_box(&buf[..]); - test::black_box(&buf[..]); - test::black_box(&buf[..]); - test::black_box(&buf[..]); - test::black_box(&buf[..]); - } - }) -} - #[bench] fn deref_shared(b: &mut Bencher) { - let mut buf = BytesMut::with_capacity(4096); - buf.put(&[0u8; 1024][..]); - let _b2 = buf.split_off(1024); + let buf = Bytes::from(vec![0; 1024]); + let _b2 = buf.clone(); b.iter(|| { for _ in 0..1024 { @@ -86,9 +30,8 @@ fn deref_shared(b: &mut Bencher) { } #[bench] -fn deref_inline(b: &mut Bencher) { - let mut buf = BytesMut::with_capacity(8); - buf.put(&[0u8; 8][..]); +fn deref_static(b: &mut Bencher) { + let buf = Bytes::from_static(b"hello world"); b.iter(|| { for _ in 0..1024 { @@ -97,33 +40,6 @@ fn deref_inline(b: &mut Bencher) { }) } -#[bench] -fn deref_two(b: &mut Bencher) { - let mut buf1 = BytesMut::with_capacity(8); - buf1.put(&[0u8; 8][..]); - - let mut buf2 = BytesMut::with_capacity(4096); - buf2.put(&[0u8; 1024][..]); - - b.iter(|| { - for _ in 0..512 { - test::black_box(&buf1[..]); - test::black_box(&buf2[..]); - } - }) -} - -#[bench] -fn clone_inline(b: &mut Bencher) { - let bytes = Bytes::from_static(b"hello world"); - - b.iter(|| { - for _ in 0..1024 { - test::black_box(&bytes.clone()); - } - }) -} - #[bench] fn clone_static(b: &mut Bencher) { let bytes = Bytes::from_static("hello world 1234567890 and have a good byte 0987654321".as_bytes()); @@ -136,8 +52,8 @@ fn clone_static(b: &mut Bencher) { } #[bench] -fn clone_arc(b: &mut Bencher) { - let bytes = Bytes::from("hello world 1234567890 and have a good byte 0987654321".as_bytes()); +fn clone_shared(b: &mut Bencher) { + let bytes = Bytes::from(b"hello world 1234567890 and have a good byte 0987654321".to_vec()); b.iter(|| { for _ in 0..1024 { @@ -147,42 +63,14 @@ fn clone_arc(b: &mut Bencher) { } #[bench] -fn alloc_write_split_to_mid(b: &mut Bencher) { - b.iter(|| { - let mut buf = BytesMut::with_capacity(128); - buf.put_slice(&[0u8; 64]); - test::black_box(buf.split_to(64)); - }) -} - -#[bench] -fn drain_write_drain(b: &mut Bencher) { - let data = [0u8; 128]; +fn clone_arc_vec(b: &mut Bencher) { + use std::sync::Arc; + let bytes = Arc::new(b"hello world 1234567890 and have a good byte 0987654321".to_vec()); b.iter(|| { - let mut buf = BytesMut::with_capacity(1024); - let mut parts = Vec::with_capacity(8); - - for _ in 0..8 { - buf.put(&data[..]); - parts.push(buf.split_to(128)); + for _ in 0..1024 { + test::black_box(&bytes.clone()); } - - test::black_box(parts); - }) -} - -#[bench] -fn fmt_write(b: &mut Bencher) { - use std::fmt::Write; - let mut buf = BytesMut::with_capacity(128); - let s = "foo bar baz quux lorem ipsum dolor et"; - - b.bytes = s.len() as u64; - b.iter(|| { - let _ = write!(buf, "{}", s); - test::black_box(&buf); - unsafe { buf.set_len(0); } }) } @@ -191,7 +79,7 @@ fn from_long_slice(b: &mut Bencher) { let data = [0u8; 128]; b.bytes = data.len() as u64; b.iter(|| { - let buf = BytesMut::from(&data[..]); + let buf = Bytes::copy_from_slice(&data[..]); test::black_box(buf); }) } @@ -217,34 +105,14 @@ fn slice_short_from_arc(b: &mut Bencher) { }) } -// Keep in sync with bytes.rs -#[cfg(target_pointer_width = "64")] -const INLINE_CAP: usize = 4 * 8 - 1; -#[cfg(target_pointer_width = "32")] -const INLINE_CAP: usize = 4 * 4 - 1; - -#[bench] -fn slice_avg_le_inline_from_arc(b: &mut Bencher) { - b.iter(|| { - // `clone` is to convert to ARC - let b = Bytes::from(vec![17; 1024]).clone(); - for i in 0..1000 { - // [1, INLINE_CAP] - let len = 1 + i % (INLINE_CAP - 1); - test::black_box(b.slice(i % 10..i % 10 + len)); - } - }) -} - #[bench] -fn slice_large_le_inline_from_arc(b: &mut Bencher) { +fn split_off_and_drop(b: &mut Bencher) { b.iter(|| { - // `clone` is to convert to ARC - let b = Bytes::from(vec![17; 1024]).clone(); - for i in 0..1000 { - // [INLINE_CAP - 10, INLINE_CAP] - let len = INLINE_CAP - 9 + i % 10; - test::black_box(b.slice(i % 10..i % 10 + len)); + for _ in 0..1024 { + let v = vec![10; 200]; + let mut b = Bytes::from(v); + test::black_box(b.split_off(100)); + test::black_box(b); } }) } diff --git a/benches/bytes_mut.rs b/benches/bytes_mut.rs new file mode 100644 index 000000000..b8707bb76 --- /dev/null +++ b/benches/bytes_mut.rs @@ -0,0 +1,142 @@ +#![feature(test)] +#![deny(warnings, rust_2018_idioms)] + +extern crate test; + +use test::Bencher; +use bytes::{BufMut, BytesMut}; + +#[bench] +fn alloc_small(b: &mut Bencher) { + b.iter(|| { + for _ in 0..1024 { + test::black_box(BytesMut::with_capacity(12)); + } + }) +} + +#[bench] +fn alloc_mid(b: &mut Bencher) { + b.iter(|| { + test::black_box(BytesMut::with_capacity(128)); + }) +} + +#[bench] +fn alloc_big(b: &mut Bencher) { + b.iter(|| { + test::black_box(BytesMut::with_capacity(4096)); + }) +} + + +#[bench] +fn deref_unique(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(4096); + buf.put(&[0u8; 1024][..]); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_unique_unroll(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(4096); + buf.put(&[0u8; 1024][..]); + + b.iter(|| { + for _ in 0..128 { + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_shared(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(4096); + buf.put(&[0u8; 1024][..]); + let _b2 = buf.split_off(1024); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_two(b: &mut Bencher) { + let mut buf1 = BytesMut::with_capacity(8); + buf1.put(&[0u8; 8][..]); + + let mut buf2 = BytesMut::with_capacity(4096); + buf2.put(&[0u8; 1024][..]); + + b.iter(|| { + for _ in 0..512 { + test::black_box(&buf1[..]); + test::black_box(&buf2[..]); + } + }) +} + +#[bench] +fn clone_frozen(b: &mut Bencher) { + let bytes = BytesMut::from(&b"hello world 1234567890 and have a good byte 0987654321"[..]).split().freeze(); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&bytes.clone()); + } + }) +} + +#[bench] +fn alloc_write_split_to_mid(b: &mut Bencher) { + b.iter(|| { + let mut buf = BytesMut::with_capacity(128); + buf.put_slice(&[0u8; 64]); + test::black_box(buf.split_to(64)); + }) +} + +#[bench] +fn drain_write_drain(b: &mut Bencher) { + let data = [0u8; 128]; + + b.iter(|| { + let mut buf = BytesMut::with_capacity(1024); + let mut parts = Vec::with_capacity(8); + + for _ in 0..8 { + buf.put(&data[..]); + parts.push(buf.split_to(128)); + } + + test::black_box(parts); + }) +} + +#[bench] +fn fmt_write(b: &mut Bencher) { + use std::fmt::Write; + let mut buf = BytesMut::with_capacity(128); + let s = "foo bar baz quux lorem ipsum dolor et"; + + b.bytes = s.len() as u64; + b.iter(|| { + let _ = write!(buf, "{}", s); + test::black_box(&buf); + unsafe { buf.set_len(0); } + }) +} diff --git a/ci/azure-cross-compile.yml b/ci/azure-cross-compile.yml index bddb59b28..be46ca346 100644 --- a/ci/azure-cross-compile.yml +++ b/ci/azure-cross-compile.yml @@ -1,5 +1,5 @@ parameters: - cmd: test + cmd: build rust_version: stable jobs: diff --git a/ci/azure-tsan.yml b/ci/azure-tsan.yml index 44cd5f767..198b187ab 100644 --- a/ci/azure-tsan.yml +++ b/ci/azure-tsan.yml @@ -18,9 +18,9 @@ jobs: # Run address sanitizer RUSTFLAGS="-Z sanitizer=address" \ - cargo test --tests --target x86_64-unknown-linux-gnu + cargo test --target x86_64-unknown-linux-gnu --test test_bytes --test test_buf --test test_buf_mut # Run thread sanitizer RUSTFLAGS="-Z sanitizer=thread" \ - cargo test --tests --target x86_64-unknown-linux-gnu + cargo test --target x86_64-unknown-linux-gnu --test test_bytes --test test_buf --test test_buf_mut displayName: TSAN / MSAN diff --git a/ci/tsan b/ci/tsan index 9cc54841b..e53f9b893 100644 --- a/ci/tsan +++ b/ci/tsan @@ -19,10 +19,6 @@ race:test::run_tests_console::*closure # Probably more fences in std. race:__call_tls_dtors -# `is_inline_or_static` is explicitly called concurrently without synchronization. -# The safety explanation can be found in a comment. -race:Inner::is_inline_or_static - # This ignores a false positive caused by `thread::park()`/`thread::unpark()`. # See: https://github.com/rust-lang/rust/pull/54806#issuecomment-436193353 race:pthread_cond_destroy diff --git a/src/buf/chain.rs b/src/buf/chain.rs index a66e3e8dc..7090ac49f 100644 --- a/src/buf/chain.rs +++ b/src/buf/chain.rs @@ -186,14 +186,6 @@ impl Buf for Chain n += self.b.bytes_vectored(&mut dst[n..]); n } - - fn to_bytes(&mut self) -> crate::Bytes { - let mut bytes: crate::BytesMut = self.a.to_bytes().try_mut() - .unwrap_or_else(|bytes| bytes.into()); - - bytes.put(&mut self.b); - bytes.freeze() - } } impl BufMut for Chain diff --git a/src/buf/vec_deque.rs b/src/buf/vec_deque.rs index 0b48b0ab8..195e6897f 100644 --- a/src/buf/vec_deque.rs +++ b/src/buf/vec_deque.rs @@ -20,22 +20,3 @@ impl Buf for VecDeque { self.drain(..cnt); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn hello_world() { - let mut buffer: VecDeque = VecDeque::new(); - buffer.extend(b"hello world"); - assert_eq!(11, buffer.remaining()); - assert_eq!(b"hello world", buffer.bytes()); - buffer.advance(6); - assert_eq!(b"world", buffer.bytes()); - buffer.extend(b" piece"); - let mut out = [0; 11]; - buffer.copy_to_slice(&mut out); - assert_eq!(b"world piece", &out[..]); - } -} diff --git a/src/bytes.rs b/src/bytes.rs index f48609a88..e07106f60 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -1,14 +1,13 @@ -use crate::{Buf, BufMut}; -use crate::buf::IntoIter; -use crate::debug; +use core::{cmp, fmt, hash, mem, ptr, slice, usize}; +use core::iter::{FromIterator}; +use core::ops::{Deref, RangeBounds}; -use core::{cmp, fmt, mem, hash, slice, ptr, usize}; -use core::ops::{Deref, DerefMut, RangeBounds}; -use core::sync::atomic::{self, AtomicUsize, AtomicPtr}; -use core::sync::atomic::Ordering::{Relaxed, Acquire, Release, AcqRel}; -use core::iter::{FromIterator, Iterator}; +use alloc::{vec::Vec, string::String, boxed::Box, borrow::Borrow}; -use alloc::{vec::Vec, string::String, boxed::Box, borrow::{Borrow, BorrowMut}}; +use crate::Buf; +use crate::buf::IntoIter; +use crate::debug; +use crate::loom::sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering}; /// A reference counted contiguous slice of memory. /// @@ -24,36 +23,22 @@ use alloc::{vec::Vec, string::String, boxed::Box, borrow::{Borrow, BorrowMut}}; /// ``` /// use bytes::Bytes; /// -/// let mut mem = Bytes::from(&b"Hello world"[..]); +/// let mut mem = Bytes::from("Hello world"); /// let a = mem.slice(0..5); /// -/// assert_eq!(&a[..], b"Hello"); +/// assert_eq!(a, "Hello"); /// /// let b = mem.split_to(6); /// -/// assert_eq!(&mem[..], b"world"); -/// assert_eq!(&b[..], b"Hello "); +/// assert_eq!(mem, "world"); +/// assert_eq!(b, "Hello "); /// ``` /// /// # Memory layout /// -/// The `Bytes` struct itself is fairly small, limited to a pointer to the -/// memory and 4 `usize` fields used to track information about which segment of -/// the underlying memory the `Bytes` handle has access to. -/// -/// The memory layout looks like this: -/// -/// ```text -/// +-------+ -/// | Bytes | -/// +-------+ -/// / \_____ -/// | \ -/// v v -/// +-----+------------------------------------+ -/// | Arc | | Data | | -/// +-----+------------------------------------+ -/// ``` +/// The `Bytes` struct itself is fairly small, limited to 4 `usize` fields used +/// to track information about which segment of the underlying memory the +/// `Bytes` handle has access to. /// /// `Bytes` keeps both a pointer to the shared `Arc` containing the full memory /// slice and a pointer to the start of the region visible by the handle. @@ -82,343 +67,22 @@ use alloc::{vec::Vec, string::String, boxed::Box, borrow::{Borrow, BorrowMut}}; /// | Arc | | | | | /// +-----+---------------------------------+-----+ /// ``` -/// -/// # Mutating -/// -/// While `Bytes` handles may potentially represent overlapping views of the -/// underlying memory slice and may not be mutated, `BytesMut` handles are -/// guaranteed to be the only handle able to view that slice of memory. As such, -/// `BytesMut` handles are able to mutate the underlying memory. Note that -/// holding a unique view to a region of memory does not mean that there are no -/// other `Bytes` and `BytesMut` handles with disjoint views of the underlying -/// memory. -/// -/// # Inline bytes -/// -/// As an optimization, when the slice referenced by a `Bytes` or `BytesMut` -/// handle is small enough [^1], `with_capacity` will avoid the allocation -/// by inlining the slice directly in the handle. In this case, a clone is no -/// longer "shallow" and the data will be copied. Converting from a `Vec` will -/// never use inlining. -/// -/// [^1]: Small enough: 31 bytes on 64 bit systems, 15 on 32 bit systems. -/// pub struct Bytes { - inner: Inner, -} - -/// A unique reference to a contiguous slice of memory. -/// -/// `BytesMut` represents a unique view into a potentially shared memory region. -/// Given the uniqueness guarantee, owners of `BytesMut` handles are able to -/// mutate the memory. It is similar to a `Vec` but with less copies and -/// allocations. -/// -/// For more detail, see [Bytes](struct.Bytes.html). -/// -/// # Growth -/// -/// One key difference from `Vec` is that most operations **do not -/// implicitly grow the buffer**. This means that calling `my_bytes.put("hello -/// world");` could panic if `my_bytes` does not have enough capacity. Before -/// writing to the buffer, ensure that there is enough remaining capacity by -/// calling `my_bytes.remaining_mut()`. In general, avoiding calls to `reserve` -/// is preferable. -/// -/// The only exception is `extend` which implicitly reserves required capacity. -/// -/// # Examples -/// -/// ``` -/// use bytes::{BytesMut, BufMut}; -/// -/// let mut buf = BytesMut::with_capacity(64); -/// -/// buf.put_u8(b'h'); -/// buf.put_u8(b'e'); -/// buf.put("llo"); -/// -/// assert_eq!(&buf[..], b"hello"); -/// -/// // Freeze the buffer so that it can be shared -/// let a = buf.freeze(); -/// -/// // This does not allocate, instead `b` points to the same memory. -/// let b = a.clone(); -/// -/// assert_eq!(&a[..], b"hello"); -/// assert_eq!(&b[..], b"hello"); -/// ``` -pub struct BytesMut { - inner: Inner, -} - -// Both `Bytes` and `BytesMut` are backed by `Inner` and functions are delegated -// to `Inner` functions. The `Bytes` and `BytesMut` shims ensure that functions -// that mutate the underlying buffer are only performed when the data range -// being mutated is only available via a single `BytesMut` handle. -// -// # Data storage modes -// -// The goal of `bytes` is to be as efficient as possible across a wide range of -// potential usage patterns. As such, `bytes` needs to be able to handle buffers -// that are never shared, shared on a single thread, and shared across many -// threads. `bytes` also needs to handle both tiny buffers as well as very large -// buffers. For example, [Cassandra](http://cassandra.apache.org) values have -// been known to be in the hundreds of megabyte, and HTTP header values can be a -// few characters in size. -// -// To achieve high performance in these various situations, `Bytes` and -// `BytesMut` use different strategies for storing the buffer depending on the -// usage pattern. -// -// ## Delayed `Arc` allocation -// -// When a `Bytes` or `BytesMut` is first created, there is only one outstanding -// handle referencing the buffer. Since sharing is not yet required, an `Arc`* is -// not used and the buffer is backed by a `Vec` directly. Using an -// `Arc>` requires two allocations, so if the buffer ends up never being -// shared, that allocation is avoided. -// -// When sharing does become necessary (`clone`, `split_to`, `split_off`), that -// is when the buffer is promoted to being shareable. The `Vec` is moved -// into an `Arc` and both the original handle and the new handle use the same -// buffer via the `Arc`. -// -// * `Arc` is being used to signify an atomically reference counted cell. We -// don't use the `Arc` implementation provided by `std` and instead use our own. -// This ends up simplifying a number of the `unsafe` code snippets. -// -// ## Inlining small buffers -// -// The `Bytes` / `BytesMut` structs require 4 pointer sized fields. On 64 bit -// systems, this ends up being 32 bytes, which is actually a lot of storage for -// cases where `Bytes` is being used to represent small byte strings, such as -// HTTP header names and values. -// -// To avoid any allocation at all in these cases, `Bytes` will use the struct -// itself for storing the buffer, reserving 1 byte for meta data. This means -// that, on 64 bit systems, 31 byte buffers require no allocation at all. -// -// The byte used for metadata stores a 2 bits flag used to indicate that the -// buffer is stored inline as well as 6 bits for tracking the buffer length (the -// return value of `Bytes::len`). -// -// ## Static buffers -// -// `Bytes` can also represent a static buffer, which is created with -// `Bytes::from_static`. No copying or allocations are required for tracking -// static buffers. The pointer to the `&'static [u8]`, the length, and a flag -// tracking that the `Bytes` instance represents a static buffer is stored in -// the `Bytes` struct. -// -// # Struct layout -// -// Both `Bytes` and `BytesMut` are wrappers around `Inner`, which provides the -// data fields as well as all of the function implementations. -// -// The `Inner` struct is carefully laid out in order to support the -// functionality described above as well as being as small as possible. Size is -// important as growing the size of the `Bytes` struct from 32 bytes to 40 bytes -// added as much as 15% overhead in benchmarks using `Bytes` in an HTTP header -// map structure. -// -// The `Inner` struct contains the following fields: -// -// * `ptr: *mut u8` -// * `len: usize` -// * `cap: usize` -// * `arc: AtomicPtr` -// -// ## `ptr: *mut u8` -// -// A pointer to start of the handle's buffer view. When backed by a `Vec`, -// this is always the `Vec`'s pointer. When backed by an `Arc>`, `ptr` -// may have been shifted to point somewhere inside the buffer. -// -// When in "inlined" mode, `ptr` is used as part of the inlined buffer. -// -// ## `len: usize` -// -// The length of the handle's buffer view. When backed by a `Vec`, this is -// always the `Vec`'s length. The slice represented by `ptr` and `len` should -// (ideally) always be initialized memory. -// -// When in "inlined" mode, `len` is used as part of the inlined buffer. -// -// ## `cap: usize` -// -// The capacity of the handle's buffer view. When backed by a `Vec`, this is -// always the `Vec`'s capacity. The slice represented by `ptr+len` and `cap-len` -// may or may not be initialized memory. -// -// When in "inlined" mode, `cap` is used as part of the inlined buffer. -// -// ## `arc: AtomicPtr` -// -// When `Inner` is in allocated mode (backed by Vec or Arc>), this -// will be the pointer to the `Arc` structure tracking the ref count for the -// underlying buffer. When the pointer is null, then the `Arc` has not been -// allocated yet and `self` is the only outstanding handle for the underlying -// buffer. -// -// The lower two bits of `arc` are used to track the storage mode of `Inner`. -// `0b01` indicates inline storage, `0b10` indicates static storage, and `0b11` -// indicates vector storage, not yet promoted to Arc. Since pointers to -// allocated structures are aligned, the lower two bits of a pointer will always -// be 0. This allows disambiguating between a pointer and the two flags. -// -// When in "inlined" mode, the least significant byte of `arc` is also used to -// store the length of the buffer view (vs. the capacity, which is a constant). -// -// The rest of `arc`'s bytes are used as part of the inline buffer, which means -// that those bytes need to be located next to the `ptr`, `len`, and `cap` -// fields, which make up the rest of the inline buffer. This requires special -// casing the layout of `Inner` depending on if the target platform is big or -// little endian. -// -// On little endian platforms, the `arc` field must be the first field in the -// struct. On big endian platforms, the `arc` field must be the last field in -// the struct. Since a deterministic struct layout is required, `Inner` is -// annotated with `#[repr(C)]`. -// -// # Thread safety -// -// `Bytes::clone()` returns a new `Bytes` handle with no copying. This is done -// by bumping the buffer ref count and returning a new struct pointing to the -// same buffer. However, the `Arc` structure is lazily allocated. This means -// that if `Bytes` is stored itself in an `Arc` (`Arc`), the `clone` -// function can be called concurrently from multiple threads. This is why an -// `AtomicPtr` is used for the `arc` field vs. a `*const`. -// -// Care is taken to ensure that the need for synchronization is minimized. Most -// operations do not require any synchronization. -// -#[cfg(target_endian = "little")] -#[repr(C)] -struct Inner { - // WARNING: Do not access the fields directly unless you know what you are - // doing. Instead, use the fns. See implementation comment above. - arc: AtomicPtr, - ptr: *mut u8, - len: usize, - cap: usize, -} - -#[cfg(target_endian = "big")] -#[repr(C)] -struct Inner { - // WARNING: Do not access the fields directly unless you know what you are - // doing. Instead, use the fns. See implementation comment above. - ptr: *mut u8, + ptr: *const u8, len: usize, - cap: usize, - arc: AtomicPtr, + // inlined "trait object" + data: AtomicPtr<()>, + vtable: &'static Vtable, } -// Thread-safe reference-counted container for the shared storage. This mostly -// the same as `core::sync::Arc` but without the weak counter. The ref counting -// fns are based on the ones found in `std`. -// -// The main reason to use `Shared` instead of `core::sync::Arc` is that it ends -// up making the overall code simpler and easier to reason about. This is due to -// some of the logic around setting `Inner::arc` and other ways the `arc` field -// is used. Using `Arc` ended up requiring a number of funky transmutes and -// other shenanigans to make it work. -struct Shared { - vec: Vec, - original_capacity_repr: usize, - ref_count: AtomicUsize, +pub(crate) struct Vtable { + /// fn(data, ptr, len) + pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes, + /// fn(data, ptr, len) + pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize), } -// Buffer storage strategy flags. -const KIND_ARC: usize = 0b00; -const KIND_INLINE: usize = 0b01; -const KIND_STATIC: usize = 0b10; -const KIND_VEC: usize = 0b11; -const KIND_MASK: usize = 0b11; - -// The max original capacity value. Any `Bytes` allocated with a greater initial -// capacity will default to this. -const MAX_ORIGINAL_CAPACITY_WIDTH: usize = 17; -// The original capacity algorithm will not take effect unless the originally -// allocated capacity was at least 1kb in size. -const MIN_ORIGINAL_CAPACITY_WIDTH: usize = 10; -// The original capacity is stored in powers of 2 starting at 1kb to a max of -// 64kb. Representing it as such requires only 3 bits of storage. -const ORIGINAL_CAPACITY_MASK: usize = 0b11100; -const ORIGINAL_CAPACITY_OFFSET: usize = 2; - -// When the storage is in the `Vec` representation, the pointer can be advanced -// at most this value. This is due to the amount of storage available to track -// the offset is usize - number of KIND bits and number of ORIGINAL_CAPACITY -// bits. -const VEC_POS_OFFSET: usize = 5; -const MAX_VEC_POS: usize = usize::MAX >> VEC_POS_OFFSET; -const NOT_VEC_POS_MASK: usize = 0b11111; - -// Bit op constants for extracting the inline length value from the `arc` field. -const INLINE_LEN_MASK: usize = 0b1111_1100; -const INLINE_LEN_OFFSET: usize = 2; - -// Byte offset from the start of `Inner` to where the inline buffer data -// starts. On little endian platforms, the first byte of the struct is the -// storage flag, so the data is shifted by a byte. On big endian systems, the -// data starts at the beginning of the struct. -#[cfg(target_endian = "little")] -const INLINE_DATA_OFFSET: isize = 1; -#[cfg(target_endian = "big")] -const INLINE_DATA_OFFSET: isize = 0; - -#[cfg(target_pointer_width = "64")] -const PTR_WIDTH: usize = 64; -#[cfg(target_pointer_width = "32")] -const PTR_WIDTH: usize = 32; - -// Inline buffer capacity. This is the size of `Inner` minus 1 byte for the -// metadata. -#[cfg(target_pointer_width = "64")] -const INLINE_CAP: usize = 4 * 8 - 1; -#[cfg(target_pointer_width = "32")] -const INLINE_CAP: usize = 4 * 4 - 1; - -/* - * - * ===== Bytes ===== - * - */ - impl Bytes { - /// Creates a new `Bytes` with the specified capacity. - /// - /// The returned `Bytes` will be able to hold at least `capacity` bytes - /// without reallocating. If `capacity` is under `4 * size_of::() - 1`, - /// then `BytesMut` will not allocate. - /// - /// It is important to note that this function does not specify the length - /// of the returned `Bytes`, but only the capacity. - /// - /// # Examples - /// - /// ``` - /// use bytes::Bytes; - /// - /// let mut bytes = Bytes::with_capacity(64); - /// - /// // `bytes` contains no data, even though there is capacity - /// assert_eq!(bytes.len(), 0); - /// - /// bytes.extend_from_slice(&b"hello world"[..]); - /// - /// assert_eq!(&bytes[..], b"hello world"); - /// ``` - #[inline] - pub fn with_capacity(capacity: usize) -> Bytes { - Bytes { - inner: Inner::with_capacity(capacity), - } - } - /// Creates a new empty `Bytes`. /// /// This will not allocate and the returned `Bytes` handle will be empty. @@ -433,7 +97,7 @@ impl Bytes { /// ``` #[inline] pub fn new() -> Bytes { - Bytes::with_capacity(0) + Bytes::from_static(b"") } /// Creates a new `Bytes` from a static slice. @@ -452,7 +116,10 @@ impl Bytes { #[inline] pub fn from_static(bytes: &'static [u8]) -> Bytes { Bytes { - inner: Inner::from_static(bytes), + ptr: bytes.as_ptr(), + len: bytes.len(), + data: AtomicPtr::new(ptr::null_mut()), + vtable: &STATIC_VTABLE, } } @@ -468,7 +135,7 @@ impl Bytes { /// ``` #[inline] pub fn len(&self) -> usize { - self.inner.len() + self.len } /// Returns true if the `Bytes` has a length of 0. @@ -483,26 +150,13 @@ impl Bytes { /// ``` #[inline] pub fn is_empty(&self) -> bool { - self.inner.is_empty() + self.len == 0 } - /// Return true if the `Bytes` uses inline allocation - /// - /// # Examples - /// ``` - /// use bytes::Bytes; - /// - /// assert!(Bytes::with_capacity(4).is_inline()); - /// assert!(!Bytes::from(Vec::with_capacity(4)).is_inline()); - /// assert!(!Bytes::with_capacity(1024).is_inline()); - /// ``` - pub fn is_inline(&self) -> bool { - self.inner.is_inline() - } ///Creates `Bytes` instance from slice, by copying it. pub fn copy_from_slice(data: &[u8]) -> Self { - BytesMut::from(data).freeze() + data.to_vec().into() } /// Returns a slice of self for the provided range. @@ -547,16 +201,15 @@ impl Bytes { assert!(begin <= end); assert!(end <= len); - if end - begin <= INLINE_CAP { - return Bytes::copy_from_slice(&self[begin..end]); + if end == begin { + return Bytes::new(); } + let mut ret = self.clone(); - unsafe { - ret.inner.set_end(end); - ret.inner.set_start(begin); - } + ret.len = end - begin; + ret.ptr = unsafe { ret.ptr.offset(begin as isize) }; ret } @@ -635,9 +288,13 @@ impl Bytes { return mem::replace(self, Bytes::new()); } - Bytes { - inner: self.inner.split_off(at), - } + let mut ret = self.clone(); + + self.len = at; + + unsafe { ret.inc_start(at) }; + + ret } /// Splits the bytes into two at the given index. @@ -674,15 +331,13 @@ impl Bytes { return Bytes::new(); } - Bytes { - inner: self.inner.split_to(at), - } - } - #[deprecated(since = "0.4.1", note = "use split_to instead")] - #[doc(hidden)] - pub fn drain_to(&mut self, at: usize) -> Bytes { - self.split_to(at) + let mut ret = self.clone(); + + unsafe { self.inc_start(at) }; + + ret.len = at; + ret } /// Shortens the buffer, keeping the first `len` bytes and dropping the @@ -705,8 +360,13 @@ impl Bytes { /// ``` /// /// [`split_off`]: #method.split_off + #[inline] pub fn truncate(&mut self, len: usize) { - self.inner.truncate(len); + if len >= self.len { + self.len = 0; + } else { + self.len = len; + } } /// Clears the buffer, removing all data. @@ -720,146 +380,64 @@ impl Bytes { /// buf.clear(); /// assert!(buf.is_empty()); /// ``` + #[inline] pub fn clear(&mut self) { self.truncate(0); } - /// Attempts to convert into a `BytesMut` handle. - /// - /// This will only succeed if there are no other outstanding references to - /// the underlying chunk of memory. `Bytes` handles that contain inlined - /// bytes will always be convertible to `BytesMut`. - /// - /// # Examples - /// - /// ``` - /// use bytes::Bytes; - /// - /// let a = Bytes::copy_from_slice(&b"Mary had a little lamb, little lamb, little lamb..."[..]); - /// - /// // Create a shallow clone - /// let b = a.clone(); - /// - /// // This will fail because `b` shares a reference with `a` - /// let a = a.try_mut().unwrap_err(); - /// - /// drop(b); - /// - /// // This will succeed - /// let mut a = a.try_mut().unwrap(); - /// - /// a[0] = b'b'; - /// - /// assert_eq!(&a[..4], b"bary"); - /// ``` - pub fn try_mut(mut self) -> Result { - if self.inner.is_mut_safe() { - Ok(BytesMut { inner: self.inner }) - } else { - Err(self) + #[inline] + pub(crate) unsafe fn with_vtable(ptr: *const u8, len: usize, data: AtomicPtr<()>, vtable: &'static Vtable) -> Bytes { + Bytes { + ptr, + len, + data, + vtable, } } - /// Acquires a mutable reference to the owned form of the data. - /// - /// Clones the data if it is not already owned. - pub fn to_mut(&mut self) -> &mut BytesMut { - if !self.inner.is_mut_safe() { - let new = Self::copy_from_slice(&self[..]); - *self = new; - } - unsafe { &mut *(self as *mut Bytes as *mut BytesMut) } - } + // private - /// Appends given bytes to this object. - /// - /// If this `Bytes` object has not enough capacity, it is resized first. - /// If it is shared (`refcount > 1`), it is copied first. - /// - /// This operation can be less effective than the similar operation on - /// `BytesMut`, especially on small additions. - /// - /// # Examples - /// - /// ``` - /// use bytes::Bytes; - /// - /// let mut buf = Bytes::from("aabb"); - /// buf.extend_from_slice(b"ccdd"); - /// buf.extend_from_slice(b"eeff"); - /// - /// assert_eq!(b"aabbccddeeff", &buf[..]); - /// ``` - pub fn extend_from_slice(&mut self, extend: &[u8]) { - if extend.is_empty() { - return; + #[inline] + fn as_slice(&self) -> &[u8] { + unsafe { + slice::from_raw_parts(self.ptr, self.len) } + } - let new_cap = self.len().checked_add(extend.len()).expect("capacity overflow"); - - let result = match mem::replace(self, Bytes::new()).try_mut() { - Ok(mut bytes_mut) => { - bytes_mut.extend_from_slice(extend); - bytes_mut - }, - Err(bytes) => { - let mut bytes_mut = BytesMut::with_capacity(new_cap); - bytes_mut.put_slice(&bytes); - bytes_mut.put_slice(extend); - bytes_mut - } - }; - - mem::replace(self, result.freeze()); + #[inline] + unsafe fn inc_start(&mut self, by: usize) { + // should already be asserted, but debug assert for tests + debug_assert!(self.len >= by); + self.len -= by; + self.ptr = self.ptr.offset(by as isize); } +} - /// Combine splitted Bytes objects back as contiguous. - /// - /// If `Bytes` objects were not contiguous originally, they will be extended. - /// - /// # Examples - /// - /// ``` - /// use bytes::Bytes; - /// - /// let mut buf = Bytes::with_capacity(64); - /// buf.extend_from_slice(b"aaabbbcccddd"); - /// - /// let splitted = buf.split_off(6); - /// assert_eq!(b"aaabbb", &buf[..]); - /// assert_eq!(b"cccddd", &splitted[..]); - /// - /// buf.unsplit(splitted); - /// assert_eq!(b"aaabbbcccddd", &buf[..]); - /// ``` - pub fn unsplit(&mut self, other: Bytes) { - if self.is_empty() { - *self = other; - return; +// Vtable must enforce this behavior +unsafe impl Send for Bytes {} +unsafe impl Sync for Bytes {} + +impl Drop for Bytes { + #[inline] + fn drop(&mut self) { + unsafe { + (self.vtable.drop)(&mut self.data, self.ptr, self.len) } + } +} - if let Err(other_inner) = self.inner.try_unsplit(other.inner) { - self.extend_from_slice(other_inner.as_ref()); +impl Clone for Bytes { + #[inline] + fn clone(&self) -> Bytes { + unsafe { + (self.vtable.clone)(&self.data, self.ptr, self.len) } } +} - /// Returns an iterator over the bytes contained by the buffer. - /// - /// # Examples - /// - /// ``` - /// use bytes::{Buf, Bytes}; - /// - /// let buf = Bytes::from(&b"abc"[..]); - /// let mut iter = buf.iter(); - /// - /// assert_eq!(iter.next().map(|b| *b), Some(b'a')); - /// assert_eq!(iter.next().map(|b| *b), Some(b'b')); - /// assert_eq!(iter.next().map(|b| *b), Some(b'c')); - /// assert_eq!(iter.next(), None); - /// ``` - pub fn iter<'a>(&'a self) -> core::slice::Iter<'a, u8> { - self.bytes().iter() +impl fmt::Debug for Bytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&debug::BsDebug(&self.as_slice()), f) } } @@ -871,28 +449,19 @@ impl Buf for Bytes { #[inline] fn bytes(&self) -> &[u8] { - &(self.inner.as_ref()) + self.as_slice() } #[inline] fn advance(&mut self, cnt: usize) { - assert!(cnt <= self.inner.as_ref().len(), "cannot advance past `remaining`"); - unsafe { self.inner.set_start(cnt); } - } -} - -impl Clone for Bytes { - fn clone(&self) -> Bytes { - Bytes { - inner: unsafe { self.inner.shallow_clone(false) }, + assert!(cnt <= self.len(), "cannot advance past `remaining`"); + unsafe { + self.inc_start(cnt); } } -} -impl AsRef<[u8]> for Bytes { - #[inline] - fn as_ref(&self) -> &[u8] { - self.inner.as_ref() + fn to_bytes(&mut self) -> crate::Bytes { + core::mem::replace(self, Bytes::new()) } } @@ -901,2111 +470,436 @@ impl Deref for Bytes { #[inline] fn deref(&self) -> &[u8] { - self.inner.as_ref() + self.as_slice() } } -impl From for Bytes { - fn from(src: BytesMut) -> Bytes { - src.freeze() +impl AsRef<[u8]> for Bytes { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_slice() } } -impl From> for Bytes { - /// Convert a `Vec` into a `Bytes` - /// - /// This constructor may be used to avoid the inlining optimization used by - /// `with_capacity`. A `Bytes` constructed this way will always store its - /// data on the heap. - fn from(src: Vec) -> Bytes { - BytesMut::from(src).freeze() +impl hash::Hash for Bytes { + fn hash(&self, state: &mut H) where H: hash::Hasher { + self.as_slice().hash(state); } } -impl From for Bytes { - fn from(src: String) -> Bytes { - BytesMut::from(src).freeze() +impl Borrow<[u8]> for Bytes { + fn borrow(&self) -> &[u8] { + self.as_slice() } } -impl From<&'static [u8]> for Bytes { - fn from(src: &'static [u8]) -> Bytes { - Bytes::from_static(src) - } -} +impl IntoIterator for Bytes { + type Item = u8; + type IntoIter = IntoIter; -impl From<&'static str> for Bytes { - fn from(src: &'static str) -> Bytes { - Bytes::from_static(src.as_bytes()) + fn into_iter(self) -> Self::IntoIter { + IntoIter::new(self) } } -impl FromIterator for BytesMut { - fn from_iter>(into_iter: T) -> Self { - let iter = into_iter.into_iter(); - let (min, maybe_max) = iter.size_hint(); - - let mut out = BytesMut::with_capacity(maybe_max.unwrap_or(min)); - - for i in iter { - out.reserve(1); - out.put_u8(i); - } +impl<'a> IntoIterator for &'a Bytes { + type Item = &'a u8; + type IntoIter = core::slice::Iter<'a, u8>; - out + fn into_iter(self) -> Self::IntoIter { + self.as_slice().into_iter() } } impl FromIterator for Bytes { fn from_iter>(into_iter: T) -> Self { - BytesMut::from_iter(into_iter).freeze() + Vec::from_iter(into_iter).into() } } -impl<'a> FromIterator<&'a u8> for BytesMut { - fn from_iter>(into_iter: T) -> Self { - BytesMut::from_iter(into_iter.into_iter().map(|b| *b)) - } -} - -impl<'a> FromIterator<&'a u8> for Bytes { - fn from_iter>(into_iter: T) -> Self { - BytesMut::from_iter(into_iter).freeze() - } -} +// impl Eq impl PartialEq for Bytes { fn eq(&self, other: &Bytes) -> bool { - self.inner.as_ref() == other.inner.as_ref() + self.as_slice() == other.as_slice() } } impl PartialOrd for Bytes { fn partial_cmp(&self, other: &Bytes) -> Option { - self.inner.as_ref().partial_cmp(other.inner.as_ref()) + self.as_slice().partial_cmp(other.as_slice()) } } impl Ord for Bytes { fn cmp(&self, other: &Bytes) -> cmp::Ordering { - self.inner.as_ref().cmp(other.inner.as_ref()) + self.as_slice().cmp(other.as_slice()) } } -impl Eq for Bytes { -} +impl Eq for Bytes {} -impl Default for Bytes { - #[inline] - fn default() -> Bytes { - Bytes::new() +impl PartialEq<[u8]> for Bytes { + fn eq(&self, other: &[u8]) -> bool { + self.as_slice() == other } } -impl fmt::Debug for Bytes { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&debug::BsDebug(&self.inner.as_ref()), fmt) +impl PartialOrd<[u8]> for Bytes { + fn partial_cmp(&self, other: &[u8]) -> Option { + self.as_slice().partial_cmp(other) } } -impl hash::Hash for Bytes { - fn hash(&self, state: &mut H) where H: hash::Hasher { - let s: &[u8] = self.as_ref(); - s.hash(state); +impl PartialEq for [u8] { + fn eq(&self, other: &Bytes) -> bool { + *other == *self } } -impl Borrow<[u8]> for Bytes { - fn borrow(&self) -> &[u8] { - self.as_ref() +impl PartialOrd for [u8] { + fn partial_cmp(&self, other: &Bytes) -> Option { + other.partial_cmp(self) } } -impl IntoIterator for Bytes { - type Item = u8; - type IntoIter = IntoIter; - - fn into_iter(self) -> Self::IntoIter { - IntoIter::new(self) +impl PartialEq for Bytes { + fn eq(&self, other: &str) -> bool { + self.as_slice() == other.as_bytes() } } -impl<'a> IntoIterator for &'a Bytes { - type Item = &'a u8; - type IntoIter = core::slice::Iter<'a, u8>; - - fn into_iter(self) -> Self::IntoIter { - self.as_ref().into_iter() +impl PartialOrd for Bytes { + fn partial_cmp(&self, other: &str) -> Option { + self.as_slice().partial_cmp(other.as_bytes()) } } -impl Extend for Bytes { - fn extend(&mut self, iter: T) where T: IntoIterator { - let iter = iter.into_iter(); - - let (lower, upper) = iter.size_hint(); - - // Avoid possible conversion into mut if there's nothing to add - if let Some(0) = upper { - return; - } +impl PartialEq for str { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} - let mut bytes_mut = match mem::replace(self, Bytes::new()).try_mut() { - Ok(bytes_mut) => bytes_mut, - Err(bytes) => { - let mut bytes_mut = BytesMut::with_capacity(bytes.len() + lower); - bytes_mut.put_slice(&bytes); - bytes_mut - } - }; +impl PartialOrd for str { + fn partial_cmp(&self, other: &Bytes) -> Option { + other.partial_cmp(self) + } +} - bytes_mut.extend(iter); +impl PartialEq> for Bytes { + fn eq(&self, other: &Vec) -> bool { + *self == &other[..] + } +} - mem::replace(self, bytes_mut.freeze()); +impl PartialOrd> for Bytes { + fn partial_cmp(&self, other: &Vec) -> Option { + self.as_slice().partial_cmp(&other[..]) } } -impl<'a> Extend<&'a u8> for Bytes { - fn extend(&mut self, iter: T) where T: IntoIterator { - self.extend(iter.into_iter().map(|b| *b)) +impl PartialEq for Vec { + fn eq(&self, other: &Bytes) -> bool { + *other == *self } } -/* - * - * ===== BytesMut ===== - * - */ - -impl BytesMut { - /// Creates a new `BytesMut` with the specified capacity. - /// - /// The returned `BytesMut` will be able to hold at least `capacity` bytes - /// without reallocating. If `capacity` is under `4 * size_of::() - 1`, - /// then `BytesMut` will not allocate. - /// - /// It is important to note that this function does not specify the length - /// of the returned `BytesMut`, but only the capacity. - /// - /// # Examples - /// - /// ``` - /// use bytes::{BytesMut, BufMut}; - /// - /// let mut bytes = BytesMut::with_capacity(64); - /// - /// // `bytes` contains no data, even though there is capacity - /// assert_eq!(bytes.len(), 0); - /// - /// bytes.put(&b"hello world"[..]); - /// - /// assert_eq!(&bytes[..], b"hello world"); - /// ``` - #[inline] - pub fn with_capacity(capacity: usize) -> BytesMut { - BytesMut { - inner: Inner::with_capacity(capacity), - } - } - - /// Creates a new `BytesMut` with default capacity. - /// - /// Resulting object has length 0 and unspecified capacity. - /// This function does not allocate. - /// - /// # Examples - /// - /// ``` - /// use bytes::{BytesMut, BufMut}; - /// - /// let mut bytes = BytesMut::new(); - /// - /// assert_eq!(0, bytes.len()); - /// - /// bytes.reserve(2); - /// bytes.put_slice(b"xy"); - /// - /// assert_eq!(&b"xy"[..], &bytes[..]); - /// ``` - #[inline] - pub fn new() -> BytesMut { - BytesMut::with_capacity(0) - } - - /// Returns the number of bytes contained in this `BytesMut`. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let b = BytesMut::from(&b"hello"[..]); - /// assert_eq!(b.len(), 5); - /// ``` - #[inline] - pub fn len(&self) -> usize { - self.inner.len() - } - - /// Returns true if the `BytesMut` has a length of 0. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let b = BytesMut::with_capacity(64); - /// assert!(b.is_empty()); - /// ``` - #[inline] - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - /// Return true if the `BytesMut` uses inline allocation - /// - /// # Examples - /// ``` - /// use bytes::BytesMut; - /// - /// assert!(BytesMut::with_capacity(4).is_inline()); - /// assert!(!BytesMut::from(Vec::with_capacity(4)).is_inline()); - /// assert!(!BytesMut::with_capacity(1024).is_inline()); - /// ``` - pub fn is_inline(&self) -> bool { - self.inner.is_inline() - } - - /// Returns the number of bytes the `BytesMut` can hold without reallocating. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let b = BytesMut::with_capacity(64); - /// assert_eq!(b.capacity(), 64); - /// ``` - #[inline] - pub fn capacity(&self) -> usize { - self.inner.capacity() - } - - /// Converts `self` into an immutable `Bytes`. - /// - /// The conversion is zero cost and is used to indicate that the slice - /// referenced by the handle will no longer be mutated. Once the conversion - /// is done, the handle can be cloned and shared across threads. - /// - /// # Examples - /// - /// ``` - /// use bytes::{BytesMut, BufMut}; - /// use std::thread; - /// - /// let mut b = BytesMut::with_capacity(64); - /// b.put("hello world"); - /// let b1 = b.freeze(); - /// let b2 = b1.clone(); - /// - /// let th = thread::spawn(move || { - /// assert_eq!(&b1[..], b"hello world"); - /// }); - /// - /// assert_eq!(&b2[..], b"hello world"); - /// th.join().unwrap(); - /// ``` - #[inline] - pub fn freeze(self) -> Bytes { - Bytes { inner: self.inner } - } - - /// Splits the bytes into two at the given index. - /// - /// Afterwards `self` contains elements `[0, at)`, and the returned - /// `BytesMut` contains elements `[at, capacity)`. - /// - /// This is an `O(1)` operation that just increases the reference count - /// and sets a few indices. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut a = BytesMut::from(&b"hello world"[..]); - /// let mut b = a.split_off(5); - /// - /// a[0] = b'j'; - /// b[0] = b'!'; - /// - /// assert_eq!(&a[..], b"jello"); - /// assert_eq!(&b[..], b"!world"); - /// ``` - /// - /// # Panics - /// - /// Panics if `at > capacity`. - pub fn split_off(&mut self, at: usize) -> BytesMut { - BytesMut { - inner: self.inner.split_off(at), - } - } - - /// Removes the bytes from the current view, returning them in a new - /// `BytesMut` handle. - /// - /// Afterwards, `self` will be empty, but will retain any additional - /// capacity that it had before the operation. This is identical to - /// `self.split_to(self.len())`. - /// - /// This is an `O(1)` operation that just increases the reference count and - /// sets a few indices. - /// - /// # Examples - /// - /// ``` - /// use bytes::{BytesMut, BufMut}; - /// - /// let mut buf = BytesMut::with_capacity(1024); - /// buf.put(&b"hello world"[..]); - /// - /// let other = buf.split(); - /// - /// assert!(buf.is_empty()); - /// assert_eq!(1013, buf.capacity()); - /// - /// assert_eq!(other, b"hello world"[..]); - /// ``` - pub fn split(&mut self) -> BytesMut { - let len = self.len(); - self.split_to(len) - } - - /// Splits the buffer into two at the given index. - /// - /// Afterwards `self` contains elements `[at, len)`, and the returned `BytesMut` - /// contains elements `[0, at)`. - /// - /// This is an `O(1)` operation that just increases the reference count and - /// sets a few indices. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut a = BytesMut::from(&b"hello world"[..]); - /// let mut b = a.split_to(5); - /// - /// a[0] = b'!'; - /// b[0] = b'j'; - /// - /// assert_eq!(&a[..], b"!world"); - /// assert_eq!(&b[..], b"jello"); - /// ``` - /// - /// # Panics - /// - /// Panics if `at > len`. - pub fn split_to(&mut self, at: usize) -> BytesMut { - assert!(at <= self.len()); - - BytesMut { - inner: self.inner.split_to(at), - } - } - - #[deprecated(since = "0.4.1", note = "use split_to instead")] - #[doc(hidden)] - pub fn drain_to(&mut self, at: usize) -> BytesMut { - self.split_to(at) - } - - /// Shortens the buffer, keeping the first `len` bytes and dropping the - /// rest. - /// - /// If `len` is greater than the buffer's current length, this has no - /// effect. - /// - /// The [`split_off`] method can emulate `truncate`, but this causes the - /// excess bytes to be returned instead of dropped. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut buf = BytesMut::from(&b"hello world"[..]); - /// buf.truncate(5); - /// assert_eq!(buf, b"hello"[..]); - /// ``` - /// - /// [`split_off`]: #method.split_off - pub fn truncate(&mut self, len: usize) { - self.inner.truncate(len); - } - - /// Clears the buffer, removing all data. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut buf = BytesMut::from(&b"hello world"[..]); - /// buf.clear(); - /// assert!(buf.is_empty()); - /// ``` - pub fn clear(&mut self) { - self.truncate(0); - } - - /// Resizes the buffer so that `len` is equal to `new_len`. - /// - /// If `new_len` is greater than `len`, the buffer is extended by the - /// difference with each additional byte set to `value`. If `new_len` is - /// less than `len`, the buffer is simply truncated. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut buf = BytesMut::new(); - /// - /// buf.resize(3, 0x1); - /// assert_eq!(&buf[..], &[0x1, 0x1, 0x1]); - /// - /// buf.resize(2, 0x2); - /// assert_eq!(&buf[..], &[0x1, 0x1]); - /// - /// buf.resize(4, 0x3); - /// assert_eq!(&buf[..], &[0x1, 0x1, 0x3, 0x3]); - /// ``` - pub fn resize(&mut self, new_len: usize, value: u8) { - self.inner.resize(new_len, value); - } - - /// Sets the length of the buffer. - /// - /// This will explicitly set the size of the buffer without actually - /// modifying the data, so it is up to the caller to ensure that the data - /// has been initialized. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut b = BytesMut::from(&b"hello world"[..]); - /// - /// unsafe { - /// b.set_len(5); - /// } - /// - /// assert_eq!(&b[..], b"hello"); - /// - /// unsafe { - /// b.set_len(11); - /// } - /// - /// assert_eq!(&b[..], b"hello world"); - /// ``` - /// - /// # Panics - /// - /// This method will panic if `len` is out of bounds for the underlying - /// slice or if it comes after the `end` of the configured window. - pub unsafe fn set_len(&mut self, len: usize) { - self.inner.set_len(len) - } - - /// Reserves capacity for at least `additional` more bytes to be inserted - /// into the given `BytesMut`. - /// - /// More than `additional` bytes may be reserved in order to avoid frequent - /// reallocations. A call to `reserve` may result in an allocation. - /// - /// Before allocating new buffer space, the function will attempt to reclaim - /// space in the existing buffer. If the current handle references a small - /// view in the original buffer and all other handles have been dropped, - /// and the requested capacity is less than or equal to the existing - /// buffer's capacity, then the current view will be copied to the front of - /// the buffer and the handle will take ownership of the full buffer. - /// - /// # Examples - /// - /// In the following example, a new buffer is allocated. - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut buf = BytesMut::from(&b"hello"[..]); - /// buf.reserve(64); - /// assert!(buf.capacity() >= 69); - /// ``` - /// - /// In the following example, the existing buffer is reclaimed. - /// - /// ``` - /// use bytes::{BytesMut, BufMut}; - /// - /// let mut buf = BytesMut::with_capacity(128); - /// buf.put(&[0; 64][..]); - /// - /// let ptr = buf.as_ptr(); - /// let other = buf.split(); - /// - /// assert!(buf.is_empty()); - /// assert_eq!(buf.capacity(), 64); - /// - /// drop(other); - /// buf.reserve(128); - /// - /// assert_eq!(buf.capacity(), 128); - /// assert_eq!(buf.as_ptr(), ptr); - /// ``` - /// - /// # Panics - /// - /// Panics if the new capacity overflows `usize`. - pub fn reserve(&mut self, additional: usize) { - self.inner.reserve(additional) - } - - /// Appends given bytes to this object. - /// - /// If this `BytesMut` object has not enough capacity, it is resized first. - /// So unlike `put_slice` operation, `extend_from_slice` does not panic. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut buf = BytesMut::with_capacity(0); - /// buf.extend_from_slice(b"aaabbb"); - /// buf.extend_from_slice(b"cccddd"); - /// - /// assert_eq!(b"aaabbbcccddd", &buf[..]); - /// ``` - pub fn extend_from_slice(&mut self, extend: &[u8]) { - self.reserve(extend.len()); - self.put_slice(extend); - } - - /// Combine splitted BytesMut objects back as contiguous. - /// - /// If `BytesMut` objects were not contiguous originally, they will be extended. - /// - /// # Examples - /// - /// ``` - /// use bytes::BytesMut; - /// - /// let mut buf = BytesMut::with_capacity(64); - /// buf.extend_from_slice(b"aaabbbcccddd"); - /// - /// let splitted = buf.split_off(6); - /// assert_eq!(b"aaabbb", &buf[..]); - /// assert_eq!(b"cccddd", &splitted[..]); - /// - /// buf.unsplit(splitted); - /// assert_eq!(b"aaabbbcccddd", &buf[..]); - /// ``` - pub fn unsplit(&mut self, other: BytesMut) { - if self.is_empty() { - *self = other; - return; - } - - if let Err(other_inner) = self.inner.try_unsplit(other.inner) { - self.extend_from_slice(other_inner.as_ref()); - } - } - - /// Returns an iterator over the bytes contained by the buffer. - /// - /// # Examples - /// - /// ``` - /// use bytes::{Buf, BytesMut}; - /// - /// let buf = BytesMut::from(&b"abc"[..]); - /// let mut iter = buf.iter(); - /// - /// assert_eq!(iter.next().map(|b| *b), Some(b'a')); - /// assert_eq!(iter.next().map(|b| *b), Some(b'b')); - /// assert_eq!(iter.next().map(|b| *b), Some(b'c')); - /// assert_eq!(iter.next(), None); - /// ``` - pub fn iter<'a>(&'a self) -> core::slice::Iter<'a, u8> { - self.bytes().iter() - } -} - -impl Buf for BytesMut { - #[inline] - fn remaining(&self) -> usize { - self.len() - } - - #[inline] - fn bytes(&self) -> &[u8] { - &(self.inner.as_ref()) - } - - #[inline] - fn advance(&mut self, cnt: usize) { - assert!(cnt <= self.inner.as_ref().len(), "cannot advance past `remaining`"); - unsafe { self.inner.set_start(cnt); } - } -} - -impl BufMut for BytesMut { - #[inline] - fn remaining_mut(&self) -> usize { - self.capacity() - self.len() - } - - #[inline] - unsafe fn advance_mut(&mut self, cnt: usize) { - let new_len = self.len() + cnt; - - // This call will panic if `cnt` is too big - self.inner.set_len(new_len); - } - - #[inline] - unsafe fn bytes_mut(&mut self) -> &mut [u8] { - let len = self.len(); - - // This will never panic as `len` can never become invalid - &mut self.inner.as_raw()[len..] - } - - #[inline] - fn put_slice(&mut self, src: &[u8]) { - assert!(self.remaining_mut() >= src.len()); - - let len = src.len(); - - unsafe { - self.bytes_mut()[..len].copy_from_slice(src); - self.advance_mut(len); - } - } - - #[inline] - fn put_u8(&mut self, n: u8) { - self.inner.put_u8(n); - } - - #[inline] - fn put_i8(&mut self, n: i8) { - self.put_u8(n as u8); - } -} - -impl AsRef<[u8]> for BytesMut { - #[inline] - fn as_ref(&self) -> &[u8] { - self.inner.as_ref() - } -} - -impl Deref for BytesMut { - type Target = [u8]; - - #[inline] - fn deref(&self) -> &[u8] { - self.as_ref() - } -} - -impl AsMut<[u8]> for BytesMut { - fn as_mut(&mut self) -> &mut [u8] { - self.inner.as_mut() - } -} - -impl DerefMut for BytesMut { - #[inline] - fn deref_mut(&mut self) -> &mut [u8] { - self.inner.as_mut() - } -} - -impl From> for BytesMut { - /// Convert a `Vec` into a `BytesMut` - /// - /// This constructor may be used to avoid the inlining optimization used by - /// `with_capacity`. A `BytesMut` constructed this way will always store - /// its data on the heap. - fn from(src: Vec) -> BytesMut { - BytesMut { - inner: Inner::from_vec(src), - } - } -} - -impl From for BytesMut { - fn from(src: String) -> BytesMut { - BytesMut::from(src.into_bytes()) - } -} - -impl<'a> From<&'a [u8]> for BytesMut { - fn from(src: &'a [u8]) -> BytesMut { - let len = src.len(); - - if len == 0 { - BytesMut::new() - } else if len <= INLINE_CAP { - unsafe { - let mut inner: Inner = mem::MaybeUninit::uninit().assume_init(); - - // Set inline mask - inner.arc = AtomicPtr::new(KIND_INLINE as *mut Shared); - inner.set_inline_len(len); - inner.as_raw()[0..len].copy_from_slice(src); - - BytesMut { - inner, - } - } - } else { - BytesMut::from(src.to_vec()) - } - } -} - -impl<'a> From<&'a str> for BytesMut { - fn from(src: &'a str) -> BytesMut { - BytesMut::from(src.as_bytes()) - } -} - -impl From for BytesMut { - fn from(src: Bytes) -> BytesMut { - src.try_mut() - .unwrap_or_else(|src| BytesMut::from(&src[..])) - } -} - -impl PartialEq for BytesMut { - fn eq(&self, other: &BytesMut) -> bool { - self.inner.as_ref() == other.inner.as_ref() - } -} - -impl PartialOrd for BytesMut { - fn partial_cmp(&self, other: &BytesMut) -> Option { - self.inner.as_ref().partial_cmp(other.inner.as_ref()) - } -} - -impl Ord for BytesMut { - fn cmp(&self, other: &BytesMut) -> cmp::Ordering { - self.inner.as_ref().cmp(other.inner.as_ref()) - } -} - -impl Eq for BytesMut { -} - -impl Default for BytesMut { - #[inline] - fn default() -> BytesMut { - BytesMut::new() - } -} - -impl fmt::Debug for BytesMut { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&debug::BsDebug(&self.inner.as_ref()), fmt) - } -} - -impl hash::Hash for BytesMut { - fn hash(&self, state: &mut H) where H: hash::Hasher { - let s: &[u8] = self.as_ref(); - s.hash(state); - } -} - -impl Borrow<[u8]> for BytesMut { - fn borrow(&self) -> &[u8] { - self.as_ref() - } -} - -impl BorrowMut<[u8]> for BytesMut { - fn borrow_mut(&mut self) -> &mut [u8] { - self.as_mut() - } -} - -impl fmt::Write for BytesMut { - #[inline] - fn write_str(&mut self, s: &str) -> fmt::Result { - if self.remaining_mut() >= s.len() { - self.put_slice(s.as_bytes()); - Ok(()) - } else { - Err(fmt::Error) - } - } - - #[inline] - fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result { - fmt::write(self, args) - } -} - -impl Clone for BytesMut { - fn clone(&self) -> BytesMut { - BytesMut::from(&self[..]) - } -} - -impl IntoIterator for BytesMut { - type Item = u8; - type IntoIter = IntoIter; - - fn into_iter(self) -> Self::IntoIter { - IntoIter::new(self) - } -} - -impl<'a> IntoIterator for &'a BytesMut { - type Item = &'a u8; - type IntoIter = core::slice::Iter<'a, u8>; - - fn into_iter(self) -> Self::IntoIter { - self.as_ref().into_iter() - } -} - -impl Extend for BytesMut { - fn extend(&mut self, iter: T) where T: IntoIterator { - let iter = iter.into_iter(); - - let (lower, _) = iter.size_hint(); - self.reserve(lower); - - for b in iter { - unsafe { - self.bytes_mut()[0] = b; - self.advance_mut(1); - } - } - } -} - -impl<'a> Extend<&'a u8> for BytesMut { - fn extend(&mut self, iter: T) where T: IntoIterator { - self.extend(iter.into_iter().map(|b| *b)) - } -} - -/* - * - * ===== Inner ===== - * - */ - -impl Inner { - #[inline] - fn from_static(bytes: &'static [u8]) -> Inner { - let ptr = bytes.as_ptr() as *mut u8; - - Inner { - // `arc` won't ever store a pointer. Instead, use it to - // track the fact that the `Bytes` handle is backed by a - // static buffer. - arc: AtomicPtr::new(KIND_STATIC as *mut Shared), - ptr, - len: bytes.len(), - cap: bytes.len(), - } - } - - #[inline] - fn from_vec(mut src: Vec) -> Inner { - let len = src.len(); - let cap = src.capacity(); - let ptr = src.as_mut_ptr(); - - mem::forget(src); - - let original_capacity_repr = original_capacity_to_repr(cap); - let arc = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC; - - Inner { - arc: AtomicPtr::new(arc as *mut Shared), - ptr, - len, - cap, - } - } - - #[inline] - fn with_capacity(capacity: usize) -> Inner { - if capacity <= INLINE_CAP { - unsafe { - // Using uninitialized memory is ~30% faster - let mut inner: Inner = mem::MaybeUninit::uninit().assume_init(); - inner.arc = AtomicPtr::new(KIND_INLINE as *mut Shared); - inner - } - } else { - Inner::from_vec(Vec::with_capacity(capacity)) - } - } - - /// Return a slice for the handle's view into the shared buffer - #[inline] - fn as_ref(&self) -> &[u8] { - unsafe { - if self.is_inline() { - slice::from_raw_parts(self.inline_ptr(), self.inline_len()) - } else { - slice::from_raw_parts(self.ptr, self.len) - } - } - } - - /// Return a mutable slice for the handle's view into the shared buffer - #[inline] - fn as_mut(&mut self) -> &mut [u8] { - debug_assert!(!self.is_static()); - - unsafe { - if self.is_inline() { - slice::from_raw_parts_mut(self.inline_ptr(), self.inline_len()) - } else { - slice::from_raw_parts_mut(self.ptr, self.len) - } - } - } - - /// Return a mutable slice for the handle's view into the shared buffer - /// including potentially uninitialized bytes. - #[inline] - unsafe fn as_raw(&mut self) -> &mut [u8] { - debug_assert!(!self.is_static()); - - if self.is_inline() { - slice::from_raw_parts_mut(self.inline_ptr(), INLINE_CAP) - } else { - slice::from_raw_parts_mut(self.ptr, self.cap) - } - } - - /// Insert a byte into the next slot and advance the len by 1. - #[inline] - fn put_u8(&mut self, n: u8) { - if self.is_inline() { - let len = self.inline_len(); - assert!(len < INLINE_CAP); - unsafe { - *self.inline_ptr().offset(len as isize) = n; - } - self.set_inline_len(len + 1); - } else { - assert!(self.len < self.cap); - unsafe { - *self.ptr.offset(self.len as isize) = n; - } - self.len += 1; - } - } - - #[inline] - fn len(&self) -> usize { - if self.is_inline() { - self.inline_len() - } else { - self.len - } - } - - /// Pointer to the start of the inline buffer - #[inline] - unsafe fn inline_ptr(&self) -> *mut u8 { - (self as *const Inner as *mut Inner as *mut u8) - .offset(INLINE_DATA_OFFSET) - } - - #[inline] - fn inline_len(&self) -> usize { - // This is undefind behavior due to a data race, but experimental - // evidence shows that it works in practice (discussion: - // https://internals.rust-lang.org/t/bit-wise-reasoning-for-atomic-accesses/8853). - let p: *const usize = unsafe { mem::transmute(&self.arc) }; - (unsafe { *p } & INLINE_LEN_MASK) >> INLINE_LEN_OFFSET - } - - /// Set the length of the inline buffer. This is done by writing to the - /// least significant byte of the `arc` field. - #[inline] - fn set_inline_len(&mut self, len: usize) { - debug_assert!(len <= INLINE_CAP); - let p = self.arc.get_mut(); - *p = ((*p as usize & !INLINE_LEN_MASK) | (len << INLINE_LEN_OFFSET)) as _; - } - - /// slice. - #[inline] - unsafe fn set_len(&mut self, len: usize) { - if self.is_inline() { - assert!(len <= INLINE_CAP); - self.set_inline_len(len); - } else { - assert!(len <= self.cap); - self.len = len; - } - } - - #[inline] - fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[inline] - fn capacity(&self) -> usize { - if self.is_inline() { - INLINE_CAP - } else { - self.cap - } - } - - fn split_off(&mut self, at: usize) -> Inner { - let mut other = unsafe { self.shallow_clone(true) }; - - unsafe { - other.set_start(at); - self.set_end(at); - } - - other - } - - fn split_to(&mut self, at: usize) -> Inner { - let mut other = unsafe { self.shallow_clone(true) }; - - unsafe { - other.set_end(at); - self.set_start(at); - } - - other - } - - fn truncate(&mut self, len: usize) { - if len <= self.len() { - unsafe { self.set_len(len); } - } - } - - fn try_unsplit(&mut self, other: Inner) -> Result<(), Inner> { - let ptr; - - if other.is_empty() { - return Ok(()); - } - - unsafe { - ptr = self.ptr.offset(self.len as isize); - } - if ptr == other.ptr && - self.kind() == KIND_ARC && - other.kind() == KIND_ARC - { - debug_assert_eq!(self.arc.load(Acquire), - other.arc.load(Acquire)); - // Contiguous blocks, just combine directly - self.len += other.len; - self.cap += other.cap; - Ok(()) - } else { - Err(other) - } - } - - fn resize(&mut self, new_len: usize, value: u8) { - let len = self.len(); - if new_len > len { - let additional = new_len - len; - self.reserve(additional); - unsafe { - let dst = self.as_raw()[len..].as_mut_ptr(); - ptr::write_bytes(dst, value, additional); - self.set_len(new_len); - } - } else { - self.truncate(new_len); - } - } - - unsafe fn set_start(&mut self, start: usize) { - // Setting the start to 0 is a no-op, so return early if this is the - // case. - if start == 0 { - return; - } - - let kind = self.kind(); - - // Always check `inline` first, because if the handle is using inline - // data storage, all of the `Inner` struct fields will be gibberish. - if kind == KIND_INLINE { - assert!(start <= INLINE_CAP); - - let len = self.inline_len(); - - if len <= start { - self.set_inline_len(0); - } else { - // `set_start` is essentially shifting data off the front of the - // view. Inlined buffers only track the length of the slice. - // So, to update the start, the data at the new starting point - // is copied to the beginning of the buffer. - let new_len = len - start; - - let dst = self.inline_ptr(); - let src = (dst as *const u8).offset(start as isize); - - ptr::copy(src, dst, new_len); - - self.set_inline_len(new_len); - } - } else { - assert!(start <= self.cap); - - if kind == KIND_VEC { - // Setting the start when in vec representation is a little more - // complicated. First, we have to track how far ahead the - // "start" of the byte buffer from the beginning of the vec. We - // also have to ensure that we don't exceed the maximum shift. - let (mut pos, prev) = self.uncoordinated_get_vec_pos(); - pos += start; - - if pos <= MAX_VEC_POS { - self.uncoordinated_set_vec_pos(pos, prev); - } else { - // The repr must be upgraded to ARC. This will never happen - // on 64 bit systems and will only happen on 32 bit systems - // when shifting past 134,217,727 bytes. As such, we don't - // worry too much about performance here. - let _ = self.shallow_clone(true); - } - } - - // Updating the start of the view is setting `ptr` to point to the - // new start and updating the `len` field to reflect the new length - // of the view. - self.ptr = self.ptr.offset(start as isize); - - if self.len >= start { - self.len -= start; - } else { - self.len = 0; - } - - self.cap -= start; - } - } - - unsafe fn set_end(&mut self, end: usize) { - debug_assert!(self.is_shared()); - - // Always check `inline` first, because if the handle is using inline - // data storage, all of the `Inner` struct fields will be gibberish. - if self.is_inline() { - assert!(end <= INLINE_CAP); - let new_len = cmp::min(self.inline_len(), end); - self.set_inline_len(new_len); - } else { - assert!(end <= self.cap); - - self.cap = end; - self.len = cmp::min(self.len, end); - } - } - - /// Checks if it is safe to mutate the memory - fn is_mut_safe(&mut self) -> bool { - let kind = self.kind(); - - // Always check `inline` first, because if the handle is using inline - // data storage, all of the `Inner` struct fields will be gibberish. - if kind == KIND_INLINE { - // Inlined buffers can always be mutated as the data is never shared - // across handles. - true - } else if kind == KIND_VEC { - true - } else if kind == KIND_STATIC { - false - } else { - // Otherwise, the underlying buffer is potentially shared with other - // handles, so the ref_count needs to be checked. - unsafe { (**self.arc.get_mut()).is_unique() } - } - } - - /// Increments the ref count. This should only be done if it is known that - /// it can be done safely. As such, this fn is not public, instead other - /// fns will use this one while maintaining the guarantees. - /// Parameter `mut_self` should only be set to `true` if caller holds - /// `&mut self` reference. - /// - /// "Safely" is defined as not exposing two `BytesMut` values that point to - /// the same byte window. - /// - /// This function is thread safe. - unsafe fn shallow_clone(&self, mut_self: bool) -> Inner { - // Always check `inline` first, because if the handle is using inline - // data storage, all of the `Inner` struct fields will be gibberish. - // - // Additionally, if kind is STATIC, then Arc is *never* changed, making - // it safe and faster to check for it now before an atomic acquire. - - if self.is_inline_or_static() { - // In this case, a shallow_clone still involves copying the data. - let mut inner: mem::MaybeUninit = mem::MaybeUninit::uninit(); - ptr::copy_nonoverlapping( - self, - inner.as_mut_ptr(), - 1, - ); - inner.assume_init() - } else { - self.shallow_clone_sync(mut_self) - } - } - - #[cold] - unsafe fn shallow_clone_sync(&self, mut_self: bool) -> Inner { - // The function requires `&self`, this means that `shallow_clone` - // could be called concurrently. - // - // The first step is to load the value of `arc`. This will determine - // how to proceed. The `Acquire` ordering synchronizes with the - // `compare_and_swap` that comes later in this function. The goal is - // to ensure that if `arc` is currently set to point to a `Shared`, - // that the current thread acquires the associated memory. - let arc = self.arc.load(Acquire); - let kind = arc as usize & KIND_MASK; - - if kind == KIND_ARC { - self.shallow_clone_arc(arc) - } else { - assert!(kind == KIND_VEC); - self.shallow_clone_vec(arc as usize, mut_self) - } - } - - unsafe fn shallow_clone_arc(&self, arc: *mut Shared) -> Inner { - debug_assert!(arc as usize & KIND_MASK == KIND_ARC); - - let old_size = (*arc).ref_count.fetch_add(1, Relaxed); - - if old_size == usize::MAX { - abort(); - } - - Inner { - arc: AtomicPtr::new(arc), - .. *self - } - } - - #[cold] - unsafe fn shallow_clone_vec(&self, arc: usize, mut_self: bool) -> Inner { - // If the buffer is still tracked in a `Vec`. It is time to - // promote the vec to an `Arc`. This could potentially be called - // concurrently, so some care must be taken. - - debug_assert!(arc & KIND_MASK == KIND_VEC); - - let original_capacity_repr = - (arc as usize & ORIGINAL_CAPACITY_MASK) >> ORIGINAL_CAPACITY_OFFSET; - - // The vec offset cannot be concurrently mutated, so there - // should be no danger reading it. - let off = (arc as usize) >> VEC_POS_OFFSET; - - // First, allocate a new `Shared` instance containing the - // `Vec` fields. It's important to note that `ptr`, `len`, - // and `cap` cannot be mutated without having `&mut self`. - // This means that these fields will not be concurrently - // updated and since the buffer hasn't been promoted to an - // `Arc`, those three fields still are the components of the - // vector. - let shared = Box::new(Shared { - vec: rebuild_vec(self.ptr, self.len, self.cap, off), - original_capacity_repr, - // Initialize refcount to 2. One for this reference, and one - // for the new clone that will be returned from - // `shallow_clone`. - ref_count: AtomicUsize::new(2), - }); - - let shared = Box::into_raw(shared); - - // The pointer should be aligned, so this assert should - // always succeed. - debug_assert!(0 == (shared as usize & 0b11)); - - // If there are no references to self in other threads, - // expensive atomic operations can be avoided. - if mut_self { - self.arc.store(shared, Relaxed); - return Inner { - arc: AtomicPtr::new(shared), - .. *self - }; - } - - // Try compare & swapping the pointer into the `arc` field. - // `Release` is used synchronize with other threads that - // will load the `arc` field. - // - // If the `compare_and_swap` fails, then the thread lost the - // race to promote the buffer to shared. The `Acquire` - // ordering will synchronize with the `compare_and_swap` - // that happened in the other thread and the `Shared` - // pointed to by `actual` will be visible. - let actual = self.arc.compare_and_swap(arc as *mut Shared, shared, AcqRel); - - if actual as usize == arc { - // The upgrade was successful, the new handle can be - // returned. - return Inner { - arc: AtomicPtr::new(shared), - .. *self - }; - } - - // The upgrade failed, a concurrent clone happened. Release - // the allocation that was made in this thread, it will not - // be needed. - let shared = Box::from_raw(shared); - mem::forget(*shared); - - // Buffer already promoted to shared storage, so increment ref - // count. - self.shallow_clone_arc(actual) - } - - #[inline] - fn reserve(&mut self, additional: usize) { - let len = self.len(); - let rem = self.capacity() - len; - - if additional <= rem { - // The handle can already store at least `additional` more bytes, so - // there is no further work needed to be done. - return; - } - - let kind = self.kind(); - - // Always check `inline` first, because if the handle is using inline - // data storage, all of the `Inner` struct fields will be gibberish. - if kind == KIND_INLINE { - let new_cap = len + additional; - - // Promote to a vector - let mut v = Vec::with_capacity(new_cap); - v.extend_from_slice(self.as_ref()); - - self.ptr = v.as_mut_ptr(); - self.len = v.len(); - self.cap = v.capacity(); - - // Since the minimum capacity is `INLINE_CAP`, don't bother encoding - // the original capacity as INLINE_CAP - self.arc = AtomicPtr::new(KIND_VEC as *mut Shared); - - mem::forget(v); - return; - } - - if kind == KIND_VEC { - // If there's enough free space before the start of the buffer, then - // just copy the data backwards and reuse the already-allocated - // space. - // - // Otherwise, since backed by a vector, use `Vec::reserve` - unsafe { - let (off, prev) = self.uncoordinated_get_vec_pos(); - - // Only reuse space if we stand to gain at least capacity/2 - // bytes of space back - if off >= additional && off >= (self.cap / 2) { - // There's space - reuse it - // - // Just move the pointer back to the start after copying - // data back. - let base_ptr = self.ptr.offset(-(off as isize)); - ptr::copy(self.ptr, base_ptr, self.len); - self.ptr = base_ptr; - self.uncoordinated_set_vec_pos(0, prev); - - // Length stays constant, but since we moved backwards we - // can gain capacity back. - self.cap += off; - } else { - // No space - allocate more - let mut v = rebuild_vec(self.ptr, self.len, self.cap, off); - v.reserve(additional); - - // Update the info - self.ptr = v.as_mut_ptr().offset(off as isize); - self.len = v.len() - off; - self.cap = v.capacity() - off; - - // Drop the vec reference - mem::forget(v); - } - return; - } - } - - let arc = *self.arc.get_mut(); - - debug_assert!(kind == KIND_ARC); - - // Reserving involves abandoning the currently shared buffer and - // allocating a new vector with the requested capacity. - // - // Compute the new capacity - let mut new_cap = len + additional; - let original_capacity; - let original_capacity_repr; - - unsafe { - original_capacity_repr = (*arc).original_capacity_repr; - original_capacity = original_capacity_from_repr(original_capacity_repr); - - // First, try to reclaim the buffer. This is possible if the current - // handle is the only outstanding handle pointing to the buffer. - if (*arc).is_unique() { - // This is the only handle to the buffer. It can be reclaimed. - // However, before doing the work of copying data, check to make - // sure that the vector has enough capacity. - let v = &mut (*arc).vec; - - if v.capacity() >= new_cap { - // The capacity is sufficient, reclaim the buffer - let ptr = v.as_mut_ptr(); - - ptr::copy(self.ptr, ptr, len); - - self.ptr = ptr; - self.cap = v.capacity(); - - return; - } - - // The vector capacity is not sufficient. The reserve request is - // asking for more than the initial buffer capacity. Allocate more - // than requested if `new_cap` is not much bigger than the current - // capacity. - // - // There are some situations, using `reserve_exact` that the - // buffer capacity could be below `original_capacity`, so do a - // check. - new_cap = cmp::max( - cmp::max(v.capacity() << 1, new_cap), - original_capacity); - } else { - new_cap = cmp::max(new_cap, original_capacity); - } - } - - // Create a new vector to store the data - let mut v = Vec::with_capacity(new_cap); - - // Copy the bytes - v.extend_from_slice(self.as_ref()); - - // Release the shared handle. This must be done *after* the bytes are - // copied. - release_shared(arc); - - // Update self - self.ptr = v.as_mut_ptr(); - self.len = v.len(); - self.cap = v.capacity(); - - let arc = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC; - - self.arc = AtomicPtr::new(arc as *mut Shared); - - // Forget the vector handle - mem::forget(v); - } - - /// Returns true if the buffer is stored inline - #[inline] - fn is_inline(&self) -> bool { - self.kind() == KIND_INLINE - } - - #[inline] - fn is_inline_or_static(&self) -> bool { - // The value returned by `kind` isn't itself safe, but the value could - // inform what operations to take, and unsafely do something without - // synchronization. - // - // KIND_INLINE and KIND_STATIC will *never* change, so branches on that - // information is safe. - let kind = self.kind(); - kind == KIND_INLINE || kind == KIND_STATIC - } - - /// Used for `debug_assert` statements. &mut is used to guarantee that it is - /// safe to check VEC_KIND - #[inline] - fn is_shared(&mut self) -> bool { - match self.kind() { - KIND_VEC => false, - _ => true, - } - } - - /// Used for `debug_assert` statements - #[inline] - fn is_static(&mut self) -> bool { - match self.kind() { - KIND_STATIC => true, - _ => false, - } - } - - #[inline] - fn kind(&self) -> usize { - // This function is going to probably raise some eyebrows. The function - // returns true if the buffer is stored inline. This is done by checking - // the least significant bit in the `arc` field. - // - // Now, you may notice that `arc` is an `AtomicPtr` and this is - // accessing it as a normal field without performing an atomic load... - // - // Again, the function only cares about the least significant bit, and - // this bit is set when `Inner` is created and never changed after that. - // All platforms have atomic "word" operations and won't randomly flip - // bits, so even without any explicit atomic operations, reading the - // flag will be correct. - // - // This is undefined behavior due to a data race, but experimental - // evidence shows that it works in practice (discussion: - // https://internals.rust-lang.org/t/bit-wise-reasoning-for-atomic-accesses/8853). - // - // This function is very critical performance wise as it is called for - // every operation. Performing an atomic load would mess with the - // compiler's ability to optimize. Simple benchmarks show up to a 10% - // slowdown using a `Relaxed` atomic load on x86. - - #[cfg(target_endian = "little")] - #[inline] - fn imp(arc: &AtomicPtr) -> usize { - unsafe { - let p: *const u8 = mem::transmute(arc); - (*p as usize) & KIND_MASK - } - } - - #[cfg(target_endian = "big")] - #[inline] - fn imp(arc: &AtomicPtr) -> usize { - unsafe { - let p: *const usize = mem::transmute(arc); - *p & KIND_MASK - } - } - - imp(&self.arc) - } - - #[inline] - fn uncoordinated_get_vec_pos(&mut self) -> (usize, usize) { - // Similar to above, this is a pretty crazed function. This should only - // be called when in the KIND_VEC mode. This + the &mut self argument - // guarantees that there is no possibility of concurrent calls to this - // function. - let prev = unsafe { - let p: &AtomicPtr = &self.arc; - let p: *const usize = mem::transmute(p); - *p - }; - - (prev >> VEC_POS_OFFSET, prev) - } - - #[inline] - fn uncoordinated_set_vec_pos(&mut self, pos: usize, prev: usize) { - // Once more... crazy - debug_assert!(pos <= MAX_VEC_POS); - - unsafe { - let p: &mut AtomicPtr = &mut self.arc; - let p: &mut usize = mem::transmute(p); - *p = (pos << VEC_POS_OFFSET) | (prev & NOT_VEC_POS_MASK); - } - } -} - -fn rebuild_vec(ptr: *mut u8, mut len: usize, mut cap: usize, off: usize) -> Vec { - unsafe { - let ptr = ptr.offset(-(off as isize)); - len += off; - cap += off; - - Vec::from_raw_parts(ptr, len, cap) - } -} - -impl Drop for Inner { - fn drop(&mut self) { - let kind = self.kind(); - - if kind == KIND_VEC { - let (off, _) = self.uncoordinated_get_vec_pos(); - - // Vector storage, free the vector - let _ = rebuild_vec(self.ptr, self.len, self.cap, off); - } else if kind == KIND_ARC { - release_shared(*self.arc.get_mut()); - } - } -} - -fn release_shared(ptr: *mut Shared) { - // `Shared` storage... follow the drop steps from Arc. - unsafe { - if (*ptr).ref_count.fetch_sub(1, Release) != 1 { - return; - } - - // This fence is needed to prevent reordering of use of the data and - // deletion of the data. Because it is marked `Release`, the decreasing - // of the reference count synchronizes with this `Acquire` fence. This - // means that use of the data happens before decreasing the reference - // count, which happens before this fence, which happens before the - // deletion of the data. - // - // As explained in the [Boost documentation][1], - // - // > It is important to enforce any possible access to the object in one - // > thread (through an existing reference) to *happen before* deleting - // > the object in a different thread. This is achieved by a "release" - // > operation after dropping a reference (any access to the object - // > through this reference must obviously happened before), and an - // > "acquire" operation before deleting the object. - // - // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) - atomic::fence(Acquire); - - // Drop the data - Box::from_raw(ptr); - } -} - -impl Shared { - fn is_unique(&self) -> bool { - // The goal is to check if the current handle is the only handle - // that currently has access to the buffer. This is done by - // checking if the `ref_count` is currently 1. - // - // The `Acquire` ordering synchronizes with the `Release` as - // part of the `fetch_sub` in `release_shared`. The `fetch_sub` - // operation guarantees that any mutations done in other threads - // are ordered before the `ref_count` is decremented. As such, - // this `Acquire` will guarantee that those mutations are - // visible to the current thread. - self.ref_count.load(Acquire) == 1 - } -} - -fn original_capacity_to_repr(cap: usize) -> usize { - let width = PTR_WIDTH - ((cap >> MIN_ORIGINAL_CAPACITY_WIDTH).leading_zeros() as usize); - cmp::min(width, MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH) -} - -fn original_capacity_from_repr(repr: usize) -> usize { - if repr == 0 { - return 0; - } - - 1 << (repr + (MIN_ORIGINAL_CAPACITY_WIDTH - 1)) -} - -#[test] -fn test_original_capacity_to_repr() { - assert_eq!(original_capacity_to_repr(0), 0); - - let max_width = 32; - - for width in 1..(max_width + 1) { - let cap = 1 << width - 1; - - let expected = if width < MIN_ORIGINAL_CAPACITY_WIDTH { - 0 - } else if width < MAX_ORIGINAL_CAPACITY_WIDTH { - width - MIN_ORIGINAL_CAPACITY_WIDTH - } else { - MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH - }; - - assert_eq!(original_capacity_to_repr(cap), expected); - - if width > 1 { - assert_eq!(original_capacity_to_repr(cap + 1), expected); - } - - // MIN_ORIGINAL_CAPACITY_WIDTH must be bigger than 7 to pass tests below - if width == MIN_ORIGINAL_CAPACITY_WIDTH + 1 { - assert_eq!(original_capacity_to_repr(cap - 24), expected - 1); - assert_eq!(original_capacity_to_repr(cap + 76), expected); - } else if width == MIN_ORIGINAL_CAPACITY_WIDTH + 2 { - assert_eq!(original_capacity_to_repr(cap - 1), expected - 1); - assert_eq!(original_capacity_to_repr(cap - 48), expected - 1); - } - } -} - -#[test] -fn test_original_capacity_from_repr() { - assert_eq!(0, original_capacity_from_repr(0)); - - let min_cap = 1 << MIN_ORIGINAL_CAPACITY_WIDTH; - - assert_eq!(min_cap, original_capacity_from_repr(1)); - assert_eq!(min_cap * 2, original_capacity_from_repr(2)); - assert_eq!(min_cap * 4, original_capacity_from_repr(3)); - assert_eq!(min_cap * 8, original_capacity_from_repr(4)); - assert_eq!(min_cap * 16, original_capacity_from_repr(5)); - assert_eq!(min_cap * 32, original_capacity_from_repr(6)); - assert_eq!(min_cap * 64, original_capacity_from_repr(7)); -} - -unsafe impl Send for Inner {} -unsafe impl Sync for Inner {} - -/* - * - * ===== PartialEq / PartialOrd ===== - * - */ - -impl PartialEq<[u8]> for BytesMut { - fn eq(&self, other: &[u8]) -> bool { - &**self == other - } -} - -impl PartialOrd<[u8]> for BytesMut { - fn partial_cmp(&self, other: &[u8]) -> Option { - (**self).partial_cmp(other) - } -} - -impl PartialEq for [u8] { - fn eq(&self, other: &BytesMut) -> bool { - *other == *self - } -} - -impl PartialOrd for [u8] { - fn partial_cmp(&self, other: &BytesMut) -> Option { +impl PartialOrd for Vec { + fn partial_cmp(&self, other: &Bytes) -> Option { other.partial_cmp(self) } } -impl PartialEq for BytesMut { - fn eq(&self, other: &str) -> bool { - &**self == other.as_bytes() +impl PartialEq for Bytes { + fn eq(&self, other: &String) -> bool { + *self == &other[..] } } -impl PartialOrd for BytesMut { - fn partial_cmp(&self, other: &str) -> Option { - (**self).partial_cmp(other.as_bytes()) +impl PartialOrd for Bytes { + fn partial_cmp(&self, other: &String) -> Option { + self.as_slice().partial_cmp(other.as_bytes()) } } -impl PartialEq for str { - fn eq(&self, other: &BytesMut) -> bool { +impl PartialEq for String { + fn eq(&self, other: &Bytes) -> bool { *other == *self } } -impl PartialOrd for str { - fn partial_cmp(&self, other: &BytesMut) -> Option { +impl PartialOrd for String { + fn partial_cmp(&self, other: &Bytes) -> Option { other.partial_cmp(self) } } -impl PartialEq> for BytesMut { - fn eq(&self, other: &Vec) -> bool { - *self == &other[..] - } -} - -impl PartialOrd> for BytesMut { - fn partial_cmp(&self, other: &Vec) -> Option { - (**self).partial_cmp(&other[..]) - } -} - -impl PartialEq for Vec { - fn eq(&self, other: &BytesMut) -> bool { +impl PartialEq for &[u8] { + fn eq(&self, other: &Bytes) -> bool { *other == *self } } -impl PartialOrd for Vec { - fn partial_cmp(&self, other: &BytesMut) -> Option { +impl PartialOrd for &[u8] { + fn partial_cmp(&self, other: &Bytes) -> Option { other.partial_cmp(self) } } -impl PartialEq for BytesMut { - fn eq(&self, other: &String) -> bool { - *self == &other[..] - } -} - -impl PartialOrd for BytesMut { - fn partial_cmp(&self, other: &String) -> Option { - (**self).partial_cmp(other.as_bytes()) - } -} - -impl PartialEq for String { - fn eq(&self, other: &BytesMut) -> bool { +impl PartialEq for &str { + fn eq(&self, other: &Bytes) -> bool { *other == *self } } -impl PartialOrd for String { - fn partial_cmp(&self, other: &BytesMut) -> Option { +impl PartialOrd for &str { + fn partial_cmp(&self, other: &Bytes) -> Option { other.partial_cmp(self) } } -impl<'a, T: ?Sized> PartialEq<&'a T> for BytesMut - where BytesMut: PartialEq +impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes + where Bytes: PartialEq { fn eq(&self, other: &&'a T) -> bool { *self == **other } } -impl<'a, T: ?Sized> PartialOrd<&'a T> for BytesMut - where BytesMut: PartialOrd +impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes + where Bytes: PartialOrd { fn partial_cmp(&self, other: &&'a T) -> Option { - self.partial_cmp(*other) + self.partial_cmp(&**other) } } -impl PartialEq for &[u8] { - fn eq(&self, other: &BytesMut) -> bool { - *other == *self - } -} +// impl From -impl PartialOrd for &[u8] { - fn partial_cmp(&self, other: &BytesMut) -> Option { - other.partial_cmp(self) +impl Default for Bytes { + #[inline] + fn default() -> Bytes { + Bytes::new() } } -impl PartialEq for &str { - fn eq(&self, other: &BytesMut) -> bool { - *other == *self +impl From<&'static [u8]> for Bytes { + fn from(slice: &'static [u8]) -> Bytes { + Bytes::from_static(slice) } } -impl PartialOrd for &str { - fn partial_cmp(&self, other: &BytesMut) -> Option { - other.partial_cmp(self) +impl From<&'static str> for Bytes { + fn from(slice: &'static str) -> Bytes { + Bytes::from_static(slice.as_bytes()) } } -impl PartialEq<[u8]> for Bytes { - fn eq(&self, other: &[u8]) -> bool { - self.inner.as_ref() == other - } -} +impl From> for Bytes { + fn from(vec: Vec) -> Bytes { + let slice = vec.into_boxed_slice(); + let len = slice.len(); + let ptr = slice.as_ptr(); + drop(Box::into_raw(slice)); -impl PartialOrd<[u8]> for Bytes { - fn partial_cmp(&self, other: &[u8]) -> Option { - self.inner.as_ref().partial_cmp(other) + let data = ptr as usize | KIND_VEC; + Bytes { + ptr, + len, + data: AtomicPtr::new(data as *mut _), + vtable: &SHARED_VTABLE, + } } } -impl PartialEq for [u8] { - fn eq(&self, other: &Bytes) -> bool { - *other == *self +impl From for Bytes { + fn from(s: String) -> Bytes { + Bytes::from(s.into_bytes()) } } -impl PartialOrd for [u8] { - fn partial_cmp(&self, other: &Bytes) -> Option { - other.partial_cmp(self) - } -} +// ===== impl Vtable ===== -impl PartialEq for Bytes { - fn eq(&self, other: &str) -> bool { - self.inner.as_ref() == other.as_bytes() +impl fmt::Debug for Vtable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Vtable") + .field("clone", &(self.clone as *const ())) + .field("drop", &(self.drop as *const ())) + .finish() } } -impl PartialOrd for Bytes { - fn partial_cmp(&self, other: &str) -> Option { - self.inner.as_ref().partial_cmp(other.as_bytes()) - } -} +// ===== impl StaticVtable ===== -impl PartialEq for str { - fn eq(&self, other: &Bytes) -> bool { - *other == *self - } -} +static STATIC_VTABLE: Vtable = Vtable { + clone: static_clone, + drop: static_drop, +}; -impl PartialOrd for str { - fn partial_cmp(&self, other: &Bytes) -> Option { - other.partial_cmp(self) - } +unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let slice = slice::from_raw_parts(ptr, len); + Bytes::from_static(slice) } -impl PartialEq> for Bytes { - fn eq(&self, other: &Vec) -> bool { - *self == &other[..] - } +unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) { + // nothing to drop for &'static [u8] } -impl PartialOrd> for Bytes { - fn partial_cmp(&self, other: &Vec) -> Option { - self.inner.as_ref().partial_cmp(&other[..]) - } -} +// ===== impl SharedVtable ===== -impl PartialEq for Vec { - fn eq(&self, other: &Bytes) -> bool { - *other == *self - } +struct Shared { + // holds vec for drop, but otherwise doesnt access it + _vec: Vec, + ref_cnt: AtomicUsize, } -impl PartialOrd for Vec { - fn partial_cmp(&self, other: &Bytes) -> Option { - other.partial_cmp(self) - } -} +static SHARED_VTABLE: Vtable = Vtable { + clone: shared_clone, + drop: shared_drop, +}; -impl PartialEq for Bytes { - fn eq(&self, other: &String) -> bool { - *self == &other[..] - } -} +const KIND_ARC: usize = 0b0; +const KIND_VEC: usize = 0b1; +const KIND_MASK: usize = 0b1; -impl PartialOrd for Bytes { - fn partial_cmp(&self, other: &String) -> Option { - self.inner.as_ref().partial_cmp(other.as_bytes()) - } -} +unsafe fn shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let shared = data.load(Ordering::Acquire); + let kind = shared as usize & KIND_MASK; -impl PartialEq for String { - fn eq(&self, other: &Bytes) -> bool { - *other == *self + if kind == KIND_ARC { + shallow_clone_arc(shared as _, ptr, len) + } else { + debug_assert_eq!(kind, KIND_VEC); + shallow_clone_vec(data, shared, ptr, len) } } -impl PartialOrd for String { - fn partial_cmp(&self, other: &Bytes) -> Option { - other.partial_cmp(self) - } -} +unsafe fn shared_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { + let shared = *data.get_mut(); + let kind = shared as usize & KIND_MASK; -impl PartialEq for &[u8] { - fn eq(&self, other: &Bytes) -> bool { - *other == *self - } -} -impl PartialOrd for &[u8] { - fn partial_cmp(&self, other: &Bytes) -> Option { - other.partial_cmp(self) - } -} + if kind == KIND_ARC { + release_shared(shared as *mut Shared); + } else { + debug_assert_eq!(kind, KIND_VEC); -impl PartialEq for &str { - fn eq(&self, other: &Bytes) -> bool { - *other == *self + drop(rebuild_vec(shared, ptr, len)); } } -impl PartialOrd for &str { - fn partial_cmp(&self, other: &Bytes) -> Option { - other.partial_cmp(self) - } -} +unsafe fn rebuild_vec(shared: *const (), offset: *const u8, len: usize) -> Vec { + debug_assert_eq!(shared as usize & KIND_MASK, KIND_VEC); -impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes - where Bytes: PartialEq -{ - fn eq(&self, other: &&'a T) -> bool { - *self == **other - } + let buf = (shared as usize & !KIND_MASK) as *mut u8; + let cap = (offset as usize - buf as usize) + len; + Vec::from_raw_parts(buf, cap, cap) } -impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes - where Bytes: PartialOrd -{ - fn partial_cmp(&self, other: &&'a T) -> Option { - self.partial_cmp(&**other) +unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes { + let old_size = (*shared).ref_cnt.fetch_add(1, Ordering::Relaxed); + + if old_size > usize::MAX >> 1 { + crate::abort(); } -} -impl PartialEq for Bytes -{ - fn eq(&self, other: &BytesMut) -> bool { - &other[..] == &self[..] + Bytes { + ptr, + len, + data: AtomicPtr::new(shared as _), + vtable: &SHARED_VTABLE, } } -impl PartialEq for BytesMut -{ - fn eq(&self, other: &Bytes) -> bool { - &other[..] == &self[..] +#[cold] +unsafe fn shallow_clone_vec(atom: &AtomicPtr<()>, ptr: *const (), offset: *const u8, len: usize) -> Bytes { + // If the buffer is still tracked in a `Vec`. It is time to + // promote the vec to an `Arc`. This could potentially be called + // concurrently, so some care must be taken. + + debug_assert_eq!(ptr as usize & KIND_MASK, KIND_VEC); + + // First, allocate a new `Shared` instance containing the + // `Vec` fields. It's important to note that `ptr`, `len`, + // and `cap` cannot be mutated without having `&mut self`. + // This means that these fields will not be concurrently + // updated and since the buffer hasn't been promoted to an + // `Arc`, those three fields still are the components of the + // vector. + let vec = rebuild_vec(ptr as *const (), offset, len); + let shared = Box::new(Shared { + _vec: vec, + // Initialize refcount to 2. One for this reference, and one + // for the new clone that will be returned from + // `shallow_clone`. + ref_cnt: AtomicUsize::new(2), + }); + + let shared = Box::into_raw(shared); + + // The pointer should be aligned, so this assert should + // always succeed. + debug_assert!(0 == (shared as usize & KIND_MASK)); + + // Try compare & swapping the pointer into the `arc` field. + // `Release` is used synchronize with other threads that + // will load the `arc` field. + // + // If the `compare_and_swap` fails, then the thread lost the + // race to promote the buffer to shared. The `Acquire` + // ordering will synchronize with the `compare_and_swap` + // that happened in the other thread and the `Shared` + // pointed to by `actual` will be visible. + let actual = atom.compare_and_swap(ptr as _, shared as _, Ordering::AcqRel); + + if actual as usize == ptr as usize { + // The upgrade was successful, the new handle can be + // returned. + return Bytes { + ptr: offset, + len, + data: AtomicPtr::new(shared as _), + vtable: &SHARED_VTABLE, + }; } -} -// While there is `std::process:abort`, it's only available in Rust 1.17, and -// our minimum supported version is currently 1.15. So, this acts as an abort -// by triggering a double panic, which always aborts in Rust. -struct Abort; + // The upgrade failed, a concurrent clone happened. Release + // the allocation that was made in this thread, it will not + // be needed. + let shared = Box::from_raw(shared); + mem::forget(*shared); -impl Drop for Abort { - fn drop(&mut self) { - panic!(); - } + // Buffer already promoted to shared storage, so increment ref + // count. + shallow_clone_arc(actual as _, offset, len) } -#[inline(never)] -#[cold] -fn abort() { - let _a = Abort; - panic!(); +unsafe fn release_shared(ptr: *mut Shared) { + // `Shared` storage... follow the drop steps from Arc. + if (*ptr).ref_cnt.fetch_sub(1, Ordering::Release) != 1 { + return; + } + + // This fence is needed to prevent reordering of use of the data and + // deletion of the data. Because it is marked `Release`, the decreasing + // of the reference count synchronizes with this `Acquire` fence. This + // means that use of the data happens before decreasing the reference + // count, which happens before this fence, which happens before the + // deletion of the data. + // + // As explained in the [Boost documentation][1], + // + // > It is important to enforce any possible access to the object in one + // > thread (through an existing reference) to *happen before* deleting + // > the object in a different thread. This is achieved by a "release" + // > operation after dropping a reference (any access to the object + // > through this reference must obviously happened before), and an + // > "acquire" operation before deleting the object. + // + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) + atomic::fence(Ordering::Acquire); + + // Drop the data + Box::from_raw(ptr); } diff --git a/src/bytes_mut.rs b/src/bytes_mut.rs new file mode 100644 index 000000000..c5642cff7 --- /dev/null +++ b/src/bytes_mut.rs @@ -0,0 +1,1429 @@ +use core::{cmp, fmt, hash, isize, mem, slice, usize}; +use core::ops::{Deref, DerefMut}; +use core::ptr::{self, NonNull}; +use core::iter::{FromIterator, Iterator}; + +use alloc::{vec::Vec, string::String, boxed::Box, borrow::{Borrow, BorrowMut}}; + +use crate::{Bytes, Buf, BufMut}; +use crate::bytes::Vtable; +use crate::buf::IntoIter; +use crate::debug; +use crate::loom::sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering}; + +/// A unique reference to a contiguous slice of memory. +/// +/// `BytesMut` represents a unique view into a potentially shared memory region. +/// Given the uniqueness guarantee, owners of `BytesMut` handles are able to +/// mutate the memory. It is similar to a `Vec` but with less copies and +/// allocations. +/// +/// # Growth +/// +/// One key difference from `Vec` is that most operations **do not +/// implicitly grow the buffer**. This means that calling `my_bytes.put("hello +/// world");` could panic if `my_bytes` does not have enough capacity. Before +/// writing to the buffer, ensure that there is enough remaining capacity by +/// calling `my_bytes.remaining_mut()`. In general, avoiding calls to `reserve` +/// is preferable. +/// +/// The only exception is `extend` which implicitly reserves required capacity. +/// +/// # Examples +/// +/// ``` +/// use bytes::{BytesMut, BufMut}; +/// +/// let mut buf = BytesMut::with_capacity(64); +/// +/// buf.put_u8(b'h'); +/// buf.put_u8(b'e'); +/// buf.put("llo"); +/// +/// assert_eq!(&buf[..], b"hello"); +/// +/// // Freeze the buffer so that it can be shared +/// let a = buf.freeze(); +/// +/// // This does not allocate, instead `b` points to the same memory. +/// let b = a.clone(); +/// +/// assert_eq!(&a[..], b"hello"); +/// assert_eq!(&b[..], b"hello"); +/// ``` +pub struct BytesMut { + ptr: NonNull, + len: usize, + cap: usize, + data: *mut Shared, +} + +// Thread-safe reference-counted container for the shared storage. This mostly +// the same as `core::sync::Arc` but without the weak counter. The ref counting +// fns are based on the ones found in `std`. +// +// The main reason to use `Shared` instead of `core::sync::Arc` is that it ends +// up making the overall code simpler and easier to reason about. This is due to +// some of the logic around setting `Inner::arc` and other ways the `arc` field +// is used. Using `Arc` ended up requiring a number of funky transmutes and +// other shenanigans to make it work. +struct Shared { + vec: Vec, + original_capacity_repr: usize, + ref_count: AtomicUsize, +} + +// Buffer storage strategy flags. +const KIND_ARC: usize = 0b0; +const KIND_VEC: usize = 0b1; +const KIND_MASK: usize = 0b1; + +// The max original capacity value. Any `Bytes` allocated with a greater initial +// capacity will default to this. +const MAX_ORIGINAL_CAPACITY_WIDTH: usize = 17; +// The original capacity algorithm will not take effect unless the originally +// allocated capacity was at least 1kb in size. +const MIN_ORIGINAL_CAPACITY_WIDTH: usize = 10; +// The original capacity is stored in powers of 2 starting at 1kb to a max of +// 64kb. Representing it as such requires only 3 bits of storage. +const ORIGINAL_CAPACITY_MASK: usize = 0b11100; +const ORIGINAL_CAPACITY_OFFSET: usize = 2; + +// When the storage is in the `Vec` representation, the pointer can be advanced +// at most this value. This is due to the amount of storage available to track +// the offset is usize - number of KIND bits and number of ORIGINAL_CAPACITY +// bits. +const VEC_POS_OFFSET: usize = 5; +const MAX_VEC_POS: usize = usize::MAX >> VEC_POS_OFFSET; +const NOT_VEC_POS_MASK: usize = 0b11111; + +#[cfg(target_pointer_width = "64")] +const PTR_WIDTH: usize = 64; +#[cfg(target_pointer_width = "32")] +const PTR_WIDTH: usize = 32; + +/* + * + * ===== BytesMut ===== + * + */ + +impl BytesMut { + /// Creates a new `BytesMut` with the specified capacity. + /// + /// The returned `BytesMut` will be able to hold at least `capacity` bytes + /// without reallocating. If `capacity` is under `4 * size_of::() - 1`, + /// then `BytesMut` will not allocate. + /// + /// It is important to note that this function does not specify the length + /// of the returned `BytesMut`, but only the capacity. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut bytes = BytesMut::with_capacity(64); + /// + /// // `bytes` contains no data, even though there is capacity + /// assert_eq!(bytes.len(), 0); + /// + /// bytes.put(&b"hello world"[..]); + /// + /// assert_eq!(&bytes[..], b"hello world"); + /// ``` + #[inline] + pub fn with_capacity(capacity: usize) -> BytesMut { + BytesMut::from_vec(Vec::with_capacity(capacity)) + } + + /// Creates a new `BytesMut` with default capacity. + /// + /// Resulting object has length 0 and unspecified capacity. + /// This function does not allocate. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut bytes = BytesMut::new(); + /// + /// assert_eq!(0, bytes.len()); + /// + /// bytes.reserve(2); + /// bytes.put_slice(b"xy"); + /// + /// assert_eq!(&b"xy"[..], &bytes[..]); + /// ``` + #[inline] + pub fn new() -> BytesMut { + BytesMut::with_capacity(0) + } + + /// Returns the number of bytes contained in this `BytesMut`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let b = BytesMut::from(&b"hello"[..]); + /// assert_eq!(b.len(), 5); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Returns true if the `BytesMut` has a length of 0. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let b = BytesMut::with_capacity(64); + /// assert!(b.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the number of bytes the `BytesMut` can hold without reallocating. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let b = BytesMut::with_capacity(64); + /// assert_eq!(b.capacity(), 64); + /// ``` + #[inline] + pub fn capacity(&self) -> usize { + self.cap + } + + /// Converts `self` into an immutable `Bytes`. + /// + /// The conversion is zero cost and is used to indicate that the slice + /// referenced by the handle will no longer be mutated. Once the conversion + /// is done, the handle can be cloned and shared across threads. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// use std::thread; + /// + /// let mut b = BytesMut::with_capacity(64); + /// b.put("hello world"); + /// let b1 = b.freeze(); + /// let b2 = b1.clone(); + /// + /// let th = thread::spawn(move || { + /// assert_eq!(&b1[..], b"hello world"); + /// }); + /// + /// assert_eq!(&b2[..], b"hello world"); + /// th.join().unwrap(); + /// ``` + #[inline] + pub fn freeze(mut self) -> Bytes { + if self.kind() == KIND_VEC { + // Just re-use `Bytes` internal Vec vtable + unsafe { + let (off, _) = self.get_vec_pos(); + let vec = rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off); + mem::forget(self); + vec.into() + } + } else { + debug_assert_eq!(self.kind(), KIND_ARC); + + let ptr = self.ptr.as_ptr(); + let len = self.len; + let data = AtomicPtr::new(self.data as _); + mem::forget(self); + unsafe { + Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) + } + } + } + + /// Splits the bytes into two at the given index. + /// + /// Afterwards `self` contains elements `[0, at)`, and the returned + /// `BytesMut` contains elements `[at, capacity)`. + /// + /// This is an `O(1)` operation that just increases the reference count + /// and sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut a = BytesMut::from(&b"hello world"[..]); + /// let mut b = a.split_off(5); + /// + /// a[0] = b'j'; + /// b[0] = b'!'; + /// + /// assert_eq!(&a[..], b"jello"); + /// assert_eq!(&b[..], b"!world"); + /// ``` + /// + /// # Panics + /// + /// Panics if `at > capacity`. + pub fn split_off(&mut self, at: usize) -> BytesMut { + assert!(at <= self.capacity()); + unsafe { + let mut other = self.shallow_clone(); + other.set_start(at); + self.set_end(at); + other + } + } + + /// Removes the bytes from the current view, returning them in a new + /// `BytesMut` handle. + /// + /// Afterwards, `self` will be empty, but will retain any additional + /// capacity that it had before the operation. This is identical to + /// `self.split_to(self.len())`. + /// + /// This is an `O(1)` operation that just increases the reference count and + /// sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut buf = BytesMut::with_capacity(1024); + /// buf.put(&b"hello world"[..]); + /// + /// let other = buf.split(); + /// + /// assert!(buf.is_empty()); + /// assert_eq!(1013, buf.capacity()); + /// + /// assert_eq!(other, b"hello world"[..]); + /// ``` + pub fn split(&mut self) -> BytesMut { + let len = self.len(); + self.split_to(len) + } + + /// Splits the buffer into two at the given index. + /// + /// Afterwards `self` contains elements `[at, len)`, and the returned `BytesMut` + /// contains elements `[0, at)`. + /// + /// This is an `O(1)` operation that just increases the reference count and + /// sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut a = BytesMut::from(&b"hello world"[..]); + /// let mut b = a.split_to(5); + /// + /// a[0] = b'!'; + /// b[0] = b'j'; + /// + /// assert_eq!(&a[..], b"!world"); + /// assert_eq!(&b[..], b"jello"); + /// ``` + /// + /// # Panics + /// + /// Panics if `at > len`. + pub fn split_to(&mut self, at: usize) -> BytesMut { + assert!(at <= self.len()); + + unsafe { + let mut other = self.shallow_clone(); + other.set_end(at); + self.set_start(at); + other + } + } + + /// Shortens the buffer, keeping the first `len` bytes and dropping the + /// rest. + /// + /// If `len` is greater than the buffer's current length, this has no + /// effect. + /// + /// The [`split_off`] method can emulate `truncate`, but this causes the + /// excess bytes to be returned instead of dropped. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::from(&b"hello world"[..]); + /// buf.truncate(5); + /// assert_eq!(buf, b"hello"[..]); + /// ``` + /// + /// [`split_off`]: #method.split_off + pub fn truncate(&mut self, len: usize) { + if len <= self.len() { + unsafe { self.set_len(len); } + } + } + + /// Clears the buffer, removing all data. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::from(&b"hello world"[..]); + /// buf.clear(); + /// assert!(buf.is_empty()); + /// ``` + pub fn clear(&mut self) { + self.truncate(0); + } + + /// Resizes the buffer so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the buffer is extended by the + /// difference with each additional byte set to `value`. If `new_len` is + /// less than `len`, the buffer is simply truncated. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::new(); + /// + /// buf.resize(3, 0x1); + /// assert_eq!(&buf[..], &[0x1, 0x1, 0x1]); + /// + /// buf.resize(2, 0x2); + /// assert_eq!(&buf[..], &[0x1, 0x1]); + /// + /// buf.resize(4, 0x3); + /// assert_eq!(&buf[..], &[0x1, 0x1, 0x3, 0x3]); + /// ``` + pub fn resize(&mut self, new_len: usize, value: u8) { + let len = self.len(); + if new_len > len { + let additional = new_len - len; + self.reserve(additional); + unsafe { + let dst = self.bytes_mut().as_mut_ptr(); + ptr::write_bytes(dst, value, additional); + self.set_len(new_len); + } + } else { + self.truncate(new_len); + } + } + + /// Sets the length of the buffer. + /// + /// This will explicitly set the size of the buffer without actually + /// modifying the data, so it is up to the caller to ensure that the data + /// has been initialized. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut b = BytesMut::from(&b"hello world"[..]); + /// + /// unsafe { + /// b.set_len(5); + /// } + /// + /// assert_eq!(&b[..], b"hello"); + /// + /// unsafe { + /// b.set_len(11); + /// } + /// + /// assert_eq!(&b[..], b"hello world"); + /// ``` + pub unsafe fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.cap); + self.len = len; + } + + /// Reserves capacity for at least `additional` more bytes to be inserted + /// into the given `BytesMut`. + /// + /// More than `additional` bytes may be reserved in order to avoid frequent + /// reallocations. A call to `reserve` may result in an allocation. + /// + /// Before allocating new buffer space, the function will attempt to reclaim + /// space in the existing buffer. If the current handle references a small + /// view in the original buffer and all other handles have been dropped, + /// and the requested capacity is less than or equal to the existing + /// buffer's capacity, then the current view will be copied to the front of + /// the buffer and the handle will take ownership of the full buffer. + /// + /// # Examples + /// + /// In the following example, a new buffer is allocated. + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::from(&b"hello"[..]); + /// buf.reserve(64); + /// assert!(buf.capacity() >= 69); + /// ``` + /// + /// In the following example, the existing buffer is reclaimed. + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut buf = BytesMut::with_capacity(128); + /// buf.put(&[0; 64][..]); + /// + /// let ptr = buf.as_ptr(); + /// let other = buf.split(); + /// + /// assert!(buf.is_empty()); + /// assert_eq!(buf.capacity(), 64); + /// + /// drop(other); + /// buf.reserve(128); + /// + /// assert_eq!(buf.capacity(), 128); + /// assert_eq!(buf.as_ptr(), ptr); + /// ``` + /// + /// # Panics + /// + /// Panics if the new capacity overflows `usize`. + pub fn reserve(&mut self, additional: usize) { + let len = self.len(); + let rem = self.capacity() - len; + + if additional <= rem { + // The handle can already store at least `additional` more bytes, so + // there is no further work needed to be done. + return; + } + + let kind = self.kind(); + + if kind == KIND_VEC { + // If there's enough free space before the start of the buffer, then + // just copy the data backwards and reuse the already-allocated + // space. + // + // Otherwise, since backed by a vector, use `Vec::reserve` + unsafe { + let (off, prev) = self.get_vec_pos(); + + // Only reuse space if we stand to gain at least capacity/2 + // bytes of space back + if off >= additional && off >= (self.cap / 2) { + // There's space - reuse it + // + // Just move the pointer back to the start after copying + // data back. + let base_ptr = self.ptr.as_ptr().offset(-(off as isize)); + ptr::copy(self.ptr.as_ptr(), base_ptr, self.len); + self.ptr = vptr(base_ptr); + self.set_vec_pos(0, prev); + + // Length stays constant, but since we moved backwards we + // can gain capacity back. + self.cap += off; + } else { + // No space - allocate more + let mut v = rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off); + v.reserve(additional); + + // Update the info + self.ptr = vptr(v.as_mut_ptr().offset(off as isize)); + self.len = v.len() - off; + self.cap = v.capacity() - off; + + // Drop the vec reference + mem::forget(v); + } + return; + } + } + + debug_assert_eq!(kind, KIND_ARC); + let shared: *mut Shared = self.data as _; + + + // Reserving involves abandoning the currently shared buffer and + // allocating a new vector with the requested capacity. + // + // Compute the new capacity + let mut new_cap = len + additional; + let original_capacity; + let original_capacity_repr; + + unsafe { + original_capacity_repr = (*shared).original_capacity_repr; + original_capacity = original_capacity_from_repr(original_capacity_repr); + + // First, try to reclaim the buffer. This is possible if the current + // handle is the only outstanding handle pointing to the buffer. + if (*shared).is_unique() { + // This is the only handle to the buffer. It can be reclaimed. + // However, before doing the work of copying data, check to make + // sure that the vector has enough capacity. + let v = &mut (*shared).vec; + + if v.capacity() >= new_cap { + // The capacity is sufficient, reclaim the buffer + let ptr = v.as_mut_ptr(); + + ptr::copy(self.ptr.as_ptr(), ptr, len); + + self.ptr = vptr(ptr); + self.cap = v.capacity(); + + return; + } + + // The vector capacity is not sufficient. The reserve request is + // asking for more than the initial buffer capacity. Allocate more + // than requested if `new_cap` is not much bigger than the current + // capacity. + // + // There are some situations, using `reserve_exact` that the + // buffer capacity could be below `original_capacity`, so do a + // check. + new_cap = cmp::max( + cmp::max(v.capacity() << 1, new_cap), + original_capacity); + } else { + new_cap = cmp::max(new_cap, original_capacity); + } + } + + // Create a new vector to store the data + let mut v = Vec::with_capacity(new_cap); + + // Copy the bytes + v.extend_from_slice(self.as_ref()); + + // Release the shared handle. This must be done *after* the bytes are + // copied. + unsafe { release_shared(shared) }; + + // Update self + let data = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC; + self.data = data as _; + self.ptr = vptr(v.as_mut_ptr()); + self.len = v.len(); + self.cap = v.capacity(); + + // Forget the vector handle + mem::forget(v); + } + /// Appends given bytes to this object. + /// + /// If this `BytesMut` object has not enough capacity, it is resized first. + /// So unlike `put_slice` operation, `extend_from_slice` does not panic. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::with_capacity(0); + /// buf.extend_from_slice(b"aaabbb"); + /// buf.extend_from_slice(b"cccddd"); + /// + /// assert_eq!(b"aaabbbcccddd", &buf[..]); + /// ``` + pub fn extend_from_slice(&mut self, extend: &[u8]) { + self.reserve(extend.len()); + self.put_slice(extend); + } + + /// Combine splitted BytesMut objects back as contiguous. + /// + /// If `BytesMut` objects were not contiguous originally, they will be extended. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::with_capacity(64); + /// buf.extend_from_slice(b"aaabbbcccddd"); + /// + /// let splitted = buf.split_off(6); + /// assert_eq!(b"aaabbb", &buf[..]); + /// assert_eq!(b"cccddd", &splitted[..]); + /// + /// buf.unsplit(splitted); + /// assert_eq!(b"aaabbbcccddd", &buf[..]); + /// ``` + pub fn unsplit(&mut self, other: BytesMut) { + if self.is_empty() { + *self = other; + return; + } + + if let Err(other) = self.try_unsplit(other) { + self.extend_from_slice(other.as_ref()); + } + } + + // private + + // For now, use a `Vec` to manage the memory for us, but we may want to + // change that in the future to some alternate allocator strategy. + // + // Thus, we don't expose an easy way to construct from a `Vec` since an + // internal change could make a simple pattern (`BytesMut::from(vec)`) + // suddenly a lot more expensive. + #[inline] + pub(crate) fn from_vec(mut vec: Vec) -> BytesMut { + let ptr = vptr(vec.as_mut_ptr()); + let len = vec.len(); + let cap = vec.capacity(); + mem::forget(vec); + + let original_capacity_repr = original_capacity_to_repr(cap); + let data = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC; + + BytesMut { + ptr, + len, + cap, + data: data as *mut _, + } + } + + #[inline] + fn as_slice(&self) -> &[u8] { + unsafe { + slice::from_raw_parts(self.ptr.as_ptr(), self.len) + } + } + + #[inline] + fn as_slice_mut(&mut self) -> &mut [u8] { + unsafe { + slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) + } + } + + unsafe fn set_start(&mut self, start: usize) { + // Setting the start to 0 is a no-op, so return early if this is the + // case. + if start == 0 { + return; + } + + debug_assert!(start <= self.cap); + + let kind = self.kind(); + + if kind == KIND_VEC { + // Setting the start when in vec representation is a little more + // complicated. First, we have to track how far ahead the + // "start" of the byte buffer from the beginning of the vec. We + // also have to ensure that we don't exceed the maximum shift. + let (mut pos, prev) = self.get_vec_pos(); + pos += start; + + if pos <= MAX_VEC_POS { + self.set_vec_pos(pos, prev); + } else { + // The repr must be upgraded to ARC. This will never happen + // on 64 bit systems and will only happen on 32 bit systems + // when shifting past 134,217,727 bytes. As such, we don't + // worry too much about performance here. + self.promote_to_shared(/*ref_count = */1); + } + } + + // Updating the start of the view is setting `ptr` to point to the + // new start and updating the `len` field to reflect the new length + // of the view. + self.ptr = vptr(self.ptr.as_ptr().offset(start as isize)); + + if self.len >= start { + self.len -= start; + } else { + self.len = 0; + } + + self.cap -= start; + } + + unsafe fn set_end(&mut self, end: usize) { + debug_assert_eq!(self.kind(), KIND_ARC); + assert!(end <= self.cap); + + self.cap = end; + self.len = cmp::min(self.len, end); + } + + fn try_unsplit(&mut self, other: BytesMut) -> Result<(), BytesMut> { + if other.is_empty() { + return Ok(()); + } + + let ptr = unsafe { self.ptr.as_ptr().offset(self.len as isize) }; + if ptr == other.ptr.as_ptr() && + self.kind() == KIND_ARC && + other.kind() == KIND_ARC && + self.data == other.data + { + // Contiguous blocks, just combine directly + self.len += other.len; + self.cap += other.cap; + Ok(()) + } else { + Err(other) + } + } + + #[inline] + fn kind(&self) -> usize { + self.data as usize & KIND_MASK + } + + unsafe fn promote_to_shared(&mut self, ref_cnt: usize) { + debug_assert_eq!(self.kind(), KIND_VEC); + debug_assert!(ref_cnt == 1 || ref_cnt == 2); + + let original_capacity_repr = + (self.data as usize & ORIGINAL_CAPACITY_MASK) >> ORIGINAL_CAPACITY_OFFSET; + + // The vec offset cannot be concurrently mutated, so there + // should be no danger reading it. + let off = (self.data as usize) >> VEC_POS_OFFSET; + + // First, allocate a new `Shared` instance containing the + // `Vec` fields. It's important to note that `ptr`, `len`, + // and `cap` cannot be mutated without having `&mut self`. + // This means that these fields will not be concurrently + // updated and since the buffer hasn't been promoted to an + // `Arc`, those three fields still are the components of the + // vector. + let shared = Box::new(Shared { + vec: rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off), + original_capacity_repr, + ref_count: AtomicUsize::new(ref_cnt), + }); + + let shared = Box::into_raw(shared); + + // The pointer should be aligned, so this assert should + // always succeed. + debug_assert_eq!(shared as usize & KIND_MASK, KIND_ARC); + + self.data = shared as _; + } + + /// Makes an exact shallow clone of `self`. + /// + /// The kind of `self` doesn't matter, but this is unsafe + /// because the clone will have the same offsets. You must + /// be sure the returned value to the user doesn't allow + /// two views into the same range. + #[inline] + unsafe fn shallow_clone(&mut self) -> BytesMut { + if self.kind() == KIND_ARC { + increment_shared(self.data); + ptr::read(self) + } else { + self.promote_to_shared(/*ref_count = */2); + ptr::read(self) + } + } + + #[inline] + unsafe fn get_vec_pos(&mut self) -> (usize, usize) { + debug_assert_eq!(self.kind(), KIND_VEC); + + let prev = self.data as usize; + (prev >> VEC_POS_OFFSET, prev) + } + + #[inline] + unsafe fn set_vec_pos(&mut self, pos: usize, prev: usize) { + debug_assert_eq!(self.kind(), KIND_VEC); + debug_assert!(pos <= MAX_VEC_POS); + + self.data = ((pos << VEC_POS_OFFSET) | (prev & NOT_VEC_POS_MASK)) as *mut _; + } +} + +impl Drop for BytesMut { + fn drop(&mut self) { + let kind = self.kind(); + + if kind == KIND_VEC { + unsafe { + let (off, _) = self.get_vec_pos(); + + // Vector storage, free the vector + let _ = rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off); + } + } else if kind == KIND_ARC { + unsafe { release_shared(self.data as _) }; + } + } +} + +impl Buf for BytesMut { + #[inline] + fn remaining(&self) -> usize { + self.len() + } + + #[inline] + fn bytes(&self) -> &[u8] { + self.as_slice() + } + + #[inline] + fn advance(&mut self, cnt: usize) { + assert!(cnt <= self.remaining(), "cannot advance past `remaining`"); + unsafe { self.set_start(cnt); } + } + + fn to_bytes(&mut self) -> crate::Bytes { + self.split().freeze() + } +} + +impl BufMut for BytesMut { + #[inline] + fn remaining_mut(&self) -> usize { + self.capacity() - self.len() + } + + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + let new_len = self.len() + cnt; + assert!(new_len <= self.cap); + self.len = new_len; + } + + #[inline] + unsafe fn bytes_mut(&mut self) -> &mut [u8] { + slice::from_raw_parts_mut(self.ptr.as_ptr().offset(self.len as isize), self.cap) + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + assert!(self.remaining_mut() >= src.len()); + + let len = src.len(); + + unsafe { + self.bytes_mut()[..len].copy_from_slice(src); + self.advance_mut(len); + } + } +} + +impl AsRef<[u8]> for BytesMut { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +impl Deref for BytesMut { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + self.as_ref() + } +} + +impl AsMut<[u8]> for BytesMut { + fn as_mut(&mut self) -> &mut [u8] { + self.as_slice_mut() + } +} + +impl DerefMut for BytesMut { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + self.as_mut() + } +} + +impl<'a> From<&'a [u8]> for BytesMut { + fn from(src: &'a [u8]) -> BytesMut { + BytesMut::from_vec(src.to_vec()) + } +} + +impl<'a> From<&'a str> for BytesMut { + fn from(src: &'a str) -> BytesMut { + BytesMut::from(src.as_bytes()) + } +} + +impl PartialEq for BytesMut { + fn eq(&self, other: &BytesMut) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl PartialOrd for BytesMut { + fn partial_cmp(&self, other: &BytesMut) -> Option { + self.as_slice().partial_cmp(other.as_slice()) + } +} + +impl Ord for BytesMut { + fn cmp(&self, other: &BytesMut) -> cmp::Ordering { + self.as_slice().cmp(other.as_slice()) + } +} + +impl Eq for BytesMut { +} + +impl Default for BytesMut { + #[inline] + fn default() -> BytesMut { + BytesMut::new() + } +} + +impl fmt::Debug for BytesMut { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&debug::BsDebug(&self.as_slice()), fmt) + } +} + +impl hash::Hash for BytesMut { + fn hash(&self, state: &mut H) where H: hash::Hasher { + let s: &[u8] = self.as_ref(); + s.hash(state); + } +} + +impl Borrow<[u8]> for BytesMut { + fn borrow(&self) -> &[u8] { + self.as_ref() + } +} + +impl BorrowMut<[u8]> for BytesMut { + fn borrow_mut(&mut self) -> &mut [u8] { + self.as_mut() + } +} + +impl fmt::Write for BytesMut { + #[inline] + fn write_str(&mut self, s: &str) -> fmt::Result { + if self.remaining_mut() >= s.len() { + self.put_slice(s.as_bytes()); + Ok(()) + } else { + Err(fmt::Error) + } + } + + #[inline] + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result { + fmt::write(self, args) + } +} + +impl Clone for BytesMut { + fn clone(&self) -> BytesMut { + BytesMut::from(&self[..]) + } +} + +impl IntoIterator for BytesMut { + type Item = u8; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter::new(self) + } +} + +impl<'a> IntoIterator for &'a BytesMut { + type Item = &'a u8; + type IntoIter = core::slice::Iter<'a, u8>; + + fn into_iter(self) -> Self::IntoIter { + self.as_ref().into_iter() + } +} + +impl Extend for BytesMut { + fn extend(&mut self, iter: T) where T: IntoIterator { + let iter = iter.into_iter(); + + let (lower, _) = iter.size_hint(); + self.reserve(lower); + + for b in iter { + unsafe { + self.bytes_mut()[0] = b; + self.advance_mut(1); + } + } + } +} + +impl<'a> Extend<&'a u8> for BytesMut { + fn extend(&mut self, iter: T) where T: IntoIterator { + self.extend(iter.into_iter().map(|b| *b)) + } +} + +impl FromIterator for BytesMut { + fn from_iter>(into_iter: T) -> Self { + BytesMut::from_vec(Vec::from_iter(into_iter)) + } +} + +impl<'a> FromIterator<&'a u8> for BytesMut { + fn from_iter>(into_iter: T) -> Self { + BytesMut::from_iter(into_iter.into_iter().map(|b| *b)) + } +} + +/* + * + * ===== Inner ===== + * + */ + +unsafe fn increment_shared(ptr: *mut Shared) { + let old_size = (*ptr).ref_count.fetch_add(1, Ordering::Relaxed); + + if old_size > isize::MAX as usize { + crate::abort(); + } +} + +unsafe fn release_shared(ptr: *mut Shared) { + // `Shared` storage... follow the drop steps from Arc. + if (*ptr).ref_count.fetch_sub(1, Ordering::Release) != 1 { + return; + } + + // This fence is needed to prevent reordering of use of the data and + // deletion of the data. Because it is marked `Release`, the decreasing + // of the reference count synchronizes with this `Acquire` fence. This + // means that use of the data happens before decreasing the reference + // count, which happens before this fence, which happens before the + // deletion of the data. + // + // As explained in the [Boost documentation][1], + // + // > It is important to enforce any possible access to the object in one + // > thread (through an existing reference) to *happen before* deleting + // > the object in a different thread. This is achieved by a "release" + // > operation after dropping a reference (any access to the object + // > through this reference must obviously happened before), and an + // > "acquire" operation before deleting the object. + // + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) + atomic::fence(Ordering::Acquire); + + // Drop the data + Box::from_raw(ptr); +} + +impl Shared { + fn is_unique(&self) -> bool { + // The goal is to check if the current handle is the only handle + // that currently has access to the buffer. This is done by + // checking if the `ref_count` is currently 1. + // + // The `Acquire` ordering synchronizes with the `Release` as + // part of the `fetch_sub` in `release_shared`. The `fetch_sub` + // operation guarantees that any mutations done in other threads + // are ordered before the `ref_count` is decremented. As such, + // this `Acquire` will guarantee that those mutations are + // visible to the current thread. + self.ref_count.load(Ordering::Acquire) == 1 + } +} + +fn original_capacity_to_repr(cap: usize) -> usize { + let width = PTR_WIDTH - ((cap >> MIN_ORIGINAL_CAPACITY_WIDTH).leading_zeros() as usize); + cmp::min(width, MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH) +} + +fn original_capacity_from_repr(repr: usize) -> usize { + if repr == 0 { + return 0; + } + + 1 << (repr + (MIN_ORIGINAL_CAPACITY_WIDTH - 1)) +} + +/* +#[test] +fn test_original_capacity_to_repr() { + assert_eq!(original_capacity_to_repr(0), 0); + + let max_width = 32; + + for width in 1..(max_width + 1) { + let cap = 1 << width - 1; + + let expected = if width < MIN_ORIGINAL_CAPACITY_WIDTH { + 0 + } else if width < MAX_ORIGINAL_CAPACITY_WIDTH { + width - MIN_ORIGINAL_CAPACITY_WIDTH + } else { + MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH + }; + + assert_eq!(original_capacity_to_repr(cap), expected); + + if width > 1 { + assert_eq!(original_capacity_to_repr(cap + 1), expected); + } + + // MIN_ORIGINAL_CAPACITY_WIDTH must be bigger than 7 to pass tests below + if width == MIN_ORIGINAL_CAPACITY_WIDTH + 1 { + assert_eq!(original_capacity_to_repr(cap - 24), expected - 1); + assert_eq!(original_capacity_to_repr(cap + 76), expected); + } else if width == MIN_ORIGINAL_CAPACITY_WIDTH + 2 { + assert_eq!(original_capacity_to_repr(cap - 1), expected - 1); + assert_eq!(original_capacity_to_repr(cap - 48), expected - 1); + } + } +} + +#[test] +fn test_original_capacity_from_repr() { + assert_eq!(0, original_capacity_from_repr(0)); + + let min_cap = 1 << MIN_ORIGINAL_CAPACITY_WIDTH; + + assert_eq!(min_cap, original_capacity_from_repr(1)); + assert_eq!(min_cap * 2, original_capacity_from_repr(2)); + assert_eq!(min_cap * 4, original_capacity_from_repr(3)); + assert_eq!(min_cap * 8, original_capacity_from_repr(4)); + assert_eq!(min_cap * 16, original_capacity_from_repr(5)); + assert_eq!(min_cap * 32, original_capacity_from_repr(6)); + assert_eq!(min_cap * 64, original_capacity_from_repr(7)); +} +*/ + +unsafe impl Send for BytesMut {} +unsafe impl Sync for BytesMut {} + +/* + * + * ===== PartialEq / PartialOrd ===== + * + */ + +impl PartialEq<[u8]> for BytesMut { + fn eq(&self, other: &[u8]) -> bool { + &**self == other + } +} + +impl PartialOrd<[u8]> for BytesMut { + fn partial_cmp(&self, other: &[u8]) -> Option { + (**self).partial_cmp(other) + } +} + +impl PartialEq for [u8] { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd for [u8] { + fn partial_cmp(&self, other: &BytesMut) -> Option { + other.partial_cmp(self) + } +} + +impl PartialEq for BytesMut { + fn eq(&self, other: &str) -> bool { + &**self == other.as_bytes() + } +} + +impl PartialOrd for BytesMut { + fn partial_cmp(&self, other: &str) -> Option { + (**self).partial_cmp(other.as_bytes()) + } +} + +impl PartialEq for str { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd for str { + fn partial_cmp(&self, other: &BytesMut) -> Option { + other.partial_cmp(self) + } +} + +impl PartialEq> for BytesMut { + fn eq(&self, other: &Vec) -> bool { + *self == &other[..] + } +} + +impl PartialOrd> for BytesMut { + fn partial_cmp(&self, other: &Vec) -> Option { + (**self).partial_cmp(&other[..]) + } +} + +impl PartialEq for Vec { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd for Vec { + fn partial_cmp(&self, other: &BytesMut) -> Option { + other.partial_cmp(self) + } +} + +impl PartialEq for BytesMut { + fn eq(&self, other: &String) -> bool { + *self == &other[..] + } +} + +impl PartialOrd for BytesMut { + fn partial_cmp(&self, other: &String) -> Option { + (**self).partial_cmp(other.as_bytes()) + } +} + +impl PartialEq for String { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd for String { + fn partial_cmp(&self, other: &BytesMut) -> Option { + other.partial_cmp(self) + } +} + +impl<'a, T: ?Sized> PartialEq<&'a T> for BytesMut + where BytesMut: PartialEq +{ + fn eq(&self, other: &&'a T) -> bool { + *self == **other + } +} + +impl<'a, T: ?Sized> PartialOrd<&'a T> for BytesMut + where BytesMut: PartialOrd +{ + fn partial_cmp(&self, other: &&'a T) -> Option { + self.partial_cmp(*other) + } +} + +impl PartialEq for &[u8] { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd for &[u8] { + fn partial_cmp(&self, other: &BytesMut) -> Option { + other.partial_cmp(self) + } +} + +impl PartialEq for &str { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd for &str { + fn partial_cmp(&self, other: &BytesMut) -> Option { + other.partial_cmp(self) + } +} + +impl PartialEq for Bytes { + fn eq(&self, other: &BytesMut) -> bool { + &other[..] == &self[..] + } +} + +impl PartialEq for BytesMut { + fn eq(&self, other: &Bytes) -> bool { + &other[..] == &self[..] + } +} + +fn vptr(ptr: *mut u8) -> NonNull { + if cfg!(debug_assertions) { + NonNull::new(ptr).expect("Vec pointer should be non-null") + } else { + unsafe { NonNull::new_unchecked(ptr) } + } +} + +unsafe fn rebuild_vec(ptr: *mut u8, mut len: usize, mut cap: usize, off: usize) -> Vec { + let ptr = ptr.offset(-(off as isize)); + len += off; + cap += off; + + Vec::from_raw_parts(ptr, len, cap) +} + +// ===== impl SharedVtable ===== + +static SHARED_VTABLE: Vtable = Vtable { + clone: shared_v_clone, + drop: shared_v_drop, +}; + +unsafe fn shared_v_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let shared = data.load(Ordering::Acquire) as *mut Shared; + increment_shared(shared); + + let data = AtomicPtr::new(shared as _); + Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) +} + +unsafe fn shared_v_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { + let shared = (*data.get_mut()) as *mut Shared; + release_shared(shared as *mut Shared); +} diff --git a/src/lib.rs b/src/lib.rs index af4dfeecb..69def08b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,10 +84,13 @@ pub use crate::buf::{ BufMut, }; +mod bytes_mut; mod bytes; mod debug; mod hex; -pub use crate::bytes::{Bytes, BytesMut}; +mod loom; +pub use crate::bytes_mut::BytesMut; +pub use crate::bytes::Bytes; // Optional Serde support #[cfg(feature = "serde")] @@ -96,3 +99,25 @@ mod serde; // Optional `Either` support #[cfg(feature = "either")] mod either; + + +#[inline(never)] +#[cold] +fn abort() -> ! { + #[cfg(feature = "std")] + { + std::process::abort(); + } + + #[cfg(not(feature = "std"))] + { + struct Abort; + impl Drop for Abort { + fn drop(&mut self) { + panic!(); + } + } + let _a = Abort; + panic!("abort"); + } +} diff --git a/src/loom.rs b/src/loom.rs new file mode 100644 index 000000000..b7c704e3a --- /dev/null +++ b/src/loom.rs @@ -0,0 +1,5 @@ +pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use core::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; + } +} diff --git a/src/serde.rs b/src/serde.rs index 6ad0331f8..11020ae7f 100644 --- a/src/serde.rs +++ b/src/serde.rs @@ -5,7 +5,7 @@ use serde::{Serialize, Serializer, Deserialize, Deserializer, de}; use super::{Bytes, BytesMut}; macro_rules! serde_impl { - ($ty:ident, $visitor_ty:ident, $from_slice:ident) => ( + ($ty:ident, $visitor_ty:ident, $from_slice:ident, $from_vec:ident) => ( impl Serialize for $ty { #[inline] fn serialize(&self, serializer: S) -> Result @@ -29,13 +29,13 @@ macro_rules! serde_impl { where V: de::SeqAccess<'de> { let len = cmp::min(seq.size_hint().unwrap_or(0), 4096); - let mut values = Vec::with_capacity(len); + let mut values: Vec = Vec::with_capacity(len); while let Some(value) = seq.next_element()? { values.push(value); } - Ok(values.into()) + Ok($ty::$from_vec(values)) } #[inline] @@ -49,7 +49,7 @@ macro_rules! serde_impl { fn visit_byte_buf(self, v: Vec) -> Result where E: de::Error { - Ok($ty::from(v)) + Ok($ty::$from_vec(v)) } #[inline] @@ -63,7 +63,7 @@ macro_rules! serde_impl { fn visit_string(self, v: String) -> Result where E: de::Error { - Ok($ty::from(v)) + Ok($ty::$from_vec(v.into_bytes())) } } @@ -78,5 +78,5 @@ macro_rules! serde_impl { ); } -serde_impl!(Bytes, BytesVisitor, copy_from_slice); -serde_impl!(BytesMut, BytesMutVisitor, from); +serde_impl!(Bytes, BytesVisitor, copy_from_slice, from); +serde_impl!(BytesMut, BytesMutVisitor, from, from_vec); diff --git a/tests/fuzz_bytes.rs b/tests/fuzz_bytes.rs new file mode 100644 index 000000000..5f894dde9 --- /dev/null +++ b/tests/fuzz_bytes.rs @@ -0,0 +1,73 @@ +// pretend to like `crate::` +extern crate alloc; +#[path = "../src/buf/mod.rs"] +#[allow(warnings)] +mod buf; +#[path = "../src/debug.rs"] +#[allow(warnings)] +mod debug; +#[path = "../src/bytes.rs"] +#[allow(warnings)] +mod bytes; +#[path = "../src/bytes_mut.rs"] +#[allow(warnings)] +mod bytes_mut; +use std::process::abort; + +use self::buf::{Buf, BufMut}; +use self::bytes::Bytes; +use self::bytes_mut::BytesMut; + +use std::sync::Arc; +use loom; +use loom::thread; + +#[test] +fn bytes_cloning_vec() { + loom::model(|| { + let a = Bytes::from(b"abcdefgh".to_vec()); + let addr = a.as_ptr() as usize; + + // test the Bytes::clone is Sync by putting it in an Arc + let a1 = Arc::new(a); + let a2 = a1.clone(); + + let t1 = thread::spawn(move || { + let b: Bytes = (*a1).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + let t2 = thread::spawn(move || { + let b: Bytes = (*a2).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + }); +} + +#[test] +fn bytes_mut_cloning_frozen() { + loom::model(|| { + let a = BytesMut::from(&b"abcdefgh"[..]).split().freeze(); + let addr = a.as_ptr() as usize; + + // test the Bytes::clone is Sync by putting it in an Arc + let a1 = Arc::new(a); + let a2 = a1.clone(); + + let t1 = thread::spawn(move || { + let b: Bytes = (*a1).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + let t2 = thread::spawn(move || { + let b: Bytes = (*a2).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + }); +} diff --git a/tests/test_buf.rs b/tests/test_buf.rs index 4d60ab5a3..b251edc7f 100644 --- a/tests/test_buf.rs +++ b/tests/test_buf.rs @@ -53,3 +53,19 @@ fn test_bufs_vec() { assert_eq!(1, buf.bytes_vectored(&mut dst[..])); } + +#[test] +fn test_vec_deque() { + use std::collections::VecDeque; + + let mut buffer: VecDeque = VecDeque::new(); + buffer.extend(b"hello world"); + assert_eq!(11, buffer.remaining()); + assert_eq!(b"hello world", buffer.bytes()); + buffer.advance(6); + assert_eq!(b"world", buffer.bytes()); + buffer.extend(b" piece"); + let mut out = [0; 11]; + buffer.copy_to_slice(&mut out); + assert_eq!(b"world piece", &out[..]); +} diff --git a/tests/test_buf_mut.rs b/tests/test_buf_mut.rs index d545d2b1d..94654b7d2 100644 --- a/tests/test_buf_mut.rs +++ b/tests/test_buf_mut.rs @@ -70,11 +70,19 @@ fn test_clone() { #[test] fn test_bufs_vec_mut() { - let mut buf = BytesMut::from(&b"hello world"[..]); let b1: &mut [u8] = &mut []; let b2: &mut [u8] = &mut []; let mut dst = [IoSliceMut::new(b1), IoSliceMut::new(b2)]; + // with no capacity + let mut buf = BytesMut::new(); + assert_eq!(buf.capacity(), 0); + unsafe { + assert_eq!(0, buf.bytes_vectored_mut(&mut dst[..])); + } + + // with capacity + let mut buf = BytesMut::with_capacity(64); unsafe { assert_eq!(1, buf.bytes_vectored_mut(&mut dst[..])); } diff --git a/tests/test_bytes.rs b/tests/test_bytes.rs index 73345231e..6b3be5988 100644 --- a/tests/test_bytes.rs +++ b/tests/test_bytes.rs @@ -5,11 +5,6 @@ use bytes::{Bytes, BytesMut, Buf, BufMut}; const LONG: &'static [u8] = b"mary had a little lamb, little lamb, little lamb"; const SHORT: &'static [u8] = b"hello world"; -fn inline_cap() -> usize { - use std::mem; - 4 * mem::size_of::() - 1 -} - fn is_sync() {} fn is_send() {} @@ -21,6 +16,35 @@ fn test_bounds() { is_send::(); } +#[test] +fn test_layout() { + use std::mem; + + assert_eq!( + mem::size_of::(), + mem::size_of::() * 4, + "Bytes size should be 4 words", + ); + assert_eq!( + mem::size_of::(), + mem::size_of::() * 4, + "BytesMut should be 4 words", + ); + + assert_eq!( + mem::size_of::(), + mem::size_of::>(), + "Bytes should be same size as Option", + ); + + assert_eq!( + mem::size_of::(), + mem::size_of::>(), + "BytesMut should be same size as Option", + ); + +} + #[test] fn from_slice() { let a = Bytes::from(&b"abcdefgh"[..]); @@ -121,14 +145,14 @@ fn slice() { #[should_panic] fn slice_oob_1() { let a = Bytes::from(&b"hello world"[..]); - a.slice(5..(inline_cap() + 1)); + a.slice(5..44); } #[test] #[should_panic] fn slice_oob_2() { let a = Bytes::from(&b"hello world"[..]); - a.slice((inline_cap() + 1)..(inline_cap() + 5)); + a.slice(44..49); } #[test] @@ -150,7 +174,7 @@ fn split_off() { #[should_panic] fn split_off_oob() { let mut hello = Bytes::from(&b"helloworld"[..]); - hello.split_off(inline_cap() + 1); + hello.split_off(44); } #[test] @@ -211,21 +235,21 @@ fn split_off_to_loop() { #[test] fn split_to_1() { - // Inline - let mut a = Bytes::from(SHORT); + // Static + let mut a = Bytes::from_static(SHORT); let b = a.split_to(4); assert_eq!(SHORT[4..], a); assert_eq!(SHORT[..4], b); // Allocated - let mut a = Bytes::from(LONG); + let mut a = Bytes::copy_from_slice(LONG); let b = a.split_to(4); assert_eq!(LONG[4..], a); assert_eq!(LONG[..4], b); - let mut a = Bytes::from(LONG); + let mut a = Bytes::copy_from_slice(LONG); let b = a.split_to(30); assert_eq!(LONG[30..], a); @@ -247,14 +271,14 @@ fn split_to_2() { #[should_panic] fn split_to_oob() { let mut hello = Bytes::from(&b"helloworld"[..]); - hello.split_to(inline_cap() + 1); + hello.split_to(33); } #[test] #[should_panic] fn split_to_oob_mut() { let mut hello = BytesMut::from(&b"helloworld"[..]); - hello.split_to(inline_cap() + 1); + hello.split_to(33); } #[test] @@ -286,6 +310,24 @@ fn split_off_to_at_gt_len() { }).is_err()); } +#[test] +fn freeze_clone_shared() { + let s = &b"abcdefgh"[..]; + let b = BytesMut::from(s).split().freeze(); + assert_eq!(b, s); + let c = b.clone(); + assert_eq!(c, s); +} + +#[test] +fn freeze_clone_unique() { + let s = &b"abcdefgh"[..]; + let b = BytesMut::from(s).freeze(); + assert_eq!(b, s); + let c = b.clone(); + assert_eq!(c, s); +} + #[test] fn fns_defined_for_bytes_mut() { let mut bytes = BytesMut::from(&b"hello world"[..]); @@ -300,24 +342,6 @@ fn fns_defined_for_bytes_mut() { #[test] fn reserve_convert() { - // Inline -> Vec - let mut bytes = BytesMut::with_capacity(8); - bytes.put("hello".as_bytes()); - bytes.reserve(40); - assert_eq!(bytes.capacity(), 45); - assert_eq!(bytes, "hello"); - - // Inline -> Inline - let mut bytes = BytesMut::with_capacity(inline_cap()); - bytes.put("abcdefghijkl".as_bytes()); - - let a = bytes.split_to(10); - bytes.reserve(inline_cap() - 3); - assert_eq!(inline_cap(), bytes.capacity()); - - assert_eq!(bytes, "kl"); - assert_eq!(a, "abcdefghij"); - // Vec -> Vec let mut bytes = BytesMut::from(LONG); bytes.reserve(64); @@ -373,19 +397,18 @@ fn reserve_max_original_capacity_value() { assert_eq!(bytes.capacity(), 64 * 1024); } -// Without either looking at the internals of the BytesMut or doing weird stuff -// with the memory allocator, there's no good way to automatically verify from -// within the program that this actually recycles memory. Instead, just exercise -// the code path to ensure that the results are correct. #[test] fn reserve_vec_recycling() { - let mut bytes = BytesMut::from(Vec::with_capacity(16)); + let mut bytes = BytesMut::with_capacity(16); assert_eq!(bytes.capacity(), 16); - bytes.put("0123456789012345".as_bytes()); + let addr = bytes.as_ptr() as usize; + bytes.put("0123456789012345"); + assert_eq!(bytes.as_ptr() as usize, addr); bytes.advance(10); assert_eq!(bytes.capacity(), 6); bytes.reserve(8); assert_eq!(bytes.capacity(), 16); + assert_eq!(bytes.as_ptr() as usize, addr); } #[test] @@ -424,15 +447,6 @@ fn reserve_in_arc_nonunique_does_not_overallocate() { assert_eq!(2001, bytes.capacity()); } -#[test] -fn inline_storage() { - let mut bytes = BytesMut::with_capacity(inline_cap()); - let zero = [0u8; 64]; - - bytes.put(&zero[0..inline_cap()]); - assert_eq!(*bytes, zero[0..inline_cap()]); -} - #[test] fn extend_mut() { let mut bytes = BytesMut::with_capacity(0); @@ -440,13 +454,6 @@ fn extend_mut() { assert_eq!(*bytes, LONG[..]); } -#[test] -fn extend_shr() { - let mut bytes = Bytes::new(); - bytes.extend(LONG); - assert_eq!(*bytes, LONG[..]); -} - #[test] fn extend_from_slice_mut() { for &i in &[3, 34] { @@ -457,16 +464,6 @@ fn extend_from_slice_mut() { } } -#[test] -fn extend_from_slice_shr() { - for &i in &[3, 34] { - let mut bytes = Bytes::new(); - bytes.extend_from_slice(&LONG[..i]); - bytes.extend_from_slice(&LONG[i..]); - assert_eq!(LONG[..], *bytes); - } -} - #[test] fn from_static() { let mut a = Bytes::from_static(b"ab"); @@ -477,22 +474,28 @@ fn from_static() { } #[test] -fn advance_inline() { - let mut a = Bytes::from(&b"hello world"[..]); +fn advance_static() { + let mut a = Bytes::from_static(b"hello world"); a.advance(6); assert_eq!(a, &b"world"[..]); } #[test] -fn advance_static() { - let mut a = Bytes::from_static(b"hello world"); +fn advance_vec() { + let mut a = Bytes::from(b"hello world boooo yah world zomg wat wat".to_vec()); + a.advance(16); + assert_eq!(a, b"o yah world zomg wat wat"[..]); + + a.advance(4); + assert_eq!(a, b"h world zomg wat wat"[..]); + a.advance(6); - assert_eq!(a, &b"world"[..]); + assert_eq!(a, b"d zomg wat wat"[..]); } #[test] -fn advance_vec() { - let mut a = BytesMut::from(b"hello world boooo yah world zomg wat wat".to_vec()); +fn advance_bytes_mut() { + let mut a = BytesMut::from("hello world boooo yah world zomg wat wat"); a.advance(16); assert_eq!(a, b"o yah world zomg wat wat"[..]); @@ -510,7 +513,7 @@ fn advance_vec() { #[test] #[should_panic] fn advance_past_len() { - let mut a = BytesMut::from(b"hello world".to_vec()); + let mut a = BytesMut::from("hello world"); a.advance(20); } @@ -563,10 +566,10 @@ fn partial_eq_bytesmut() { assert!(bytesmut != bytes2); } +/* #[test] fn bytes_unsplit_basic() { - let mut buf = Bytes::with_capacity(64); - buf.extend_from_slice(b"aaabbbcccddd"); + let buf = Bytes::from(&b"aaabbbcccddd"[..]); let splitted = buf.split_off(6); assert_eq!(b"aaabbb", &buf[..]); @@ -578,8 +581,7 @@ fn bytes_unsplit_basic() { #[test] fn bytes_unsplit_empty_other() { - let mut buf = Bytes::with_capacity(64); - buf.extend_from_slice(b"aaabbbcccddd"); + let buf = Bytes::from(&b"aaabbbcccddd"[..]); // empty other let other = Bytes::new(); @@ -600,49 +602,6 @@ fn bytes_unsplit_empty_self() { assert_eq!(b"aaabbbcccddd", &buf[..]); } -#[test] -fn bytes_unsplit_inline_arc() { - let mut buf = Bytes::with_capacity(8); //inline - buf.extend_from_slice(b"aaaabbbb"); - - let mut buf2 = Bytes::with_capacity(64); - buf2.extend_from_slice(b"ccccddddeeee"); - - buf2.split_off(8); //arc - - buf.unsplit(buf2); - assert_eq!(b"aaaabbbbccccdddd", &buf[..]); -} - -#[test] -fn bytes_unsplit_arc_inline() { - let mut buf = Bytes::with_capacity(64); - buf.extend_from_slice(b"aaaabbbbeeee"); - - buf.split_off(8); //arc - - let mut buf2 = Bytes::with_capacity(8); //inline - buf2.extend_from_slice(b"ccccdddd"); - - buf.unsplit(buf2); - assert_eq!(b"aaaabbbbccccdddd", &buf[..]); - -} - -#[test] -fn bytes_unsplit_both_inline() { - let mut buf = Bytes::with_capacity(16); //inline - buf.extend_from_slice(b"aaaabbbbccccdddd"); - - let splitted = buf.split_off(8); // both inline - assert_eq!(b"aaaabbbb", &buf[..]); - assert_eq!(b"ccccdddd", &splitted[..]); - - buf.unsplit(splitted); - assert_eq!(b"aaaabbbbccccdddd", &buf[..]); -} - - #[test] fn bytes_unsplit_arc_different() { let mut buf = Bytes::with_capacity(64); @@ -696,6 +655,7 @@ fn bytes_unsplit_overlapping_references() { assert_eq!(b"abcdefghijklmnopqrst", &buf0010[..]); assert_eq!(b"fghijklmno", &buf0515[..]); } +*/ #[test] fn bytes_mut_unsplit_basic() { @@ -734,49 +694,6 @@ fn bytes_mut_unsplit_empty_self() { assert_eq!(b"aaabbbcccddd", &buf[..]); } -#[test] -fn bytes_mut_unsplit_inline_arc() { - let mut buf = BytesMut::with_capacity(8); //inline - buf.extend_from_slice(b"aaaabbbb"); - - let mut buf2 = BytesMut::with_capacity(64); - buf2.extend_from_slice(b"ccccddddeeee"); - - buf2.split_off(8); //arc - - buf.unsplit(buf2); - assert_eq!(b"aaaabbbbccccdddd", &buf[..]); -} - -#[test] -fn bytes_mut_unsplit_arc_inline() { - let mut buf = BytesMut::with_capacity(64); - buf.extend_from_slice(b"aaaabbbbeeee"); - - buf.split_off(8); //arc - - let mut buf2 = BytesMut::with_capacity(8); //inline - buf2.extend_from_slice(b"ccccdddd"); - - buf.unsplit(buf2); - assert_eq!(b"aaaabbbbccccdddd", &buf[..]); - -} - -#[test] -fn bytes_mut_unsplit_both_inline() { - let mut buf = BytesMut::with_capacity(16); //inline - buf.extend_from_slice(b"aaaabbbbccccdddd"); - - let splitted = buf.split_off(8); // both inline - assert_eq!(b"aaaabbbb", &buf[..]); - assert_eq!(b"ccccdddd", &splitted[..]); - - buf.unsplit(splitted); - assert_eq!(b"aaaabbbbccccdddd", &buf[..]); -} - - #[test] fn bytes_mut_unsplit_arc_different() { let mut buf = BytesMut::with_capacity(64);