Refactor Bytes to use an internal vtable

Bytes is a useful tool for managing multiple slices into the same region
of memory, and the other things it used to have been removed to reduce
complexity. The exact strategy for managing the multiple references is
no longer hard-coded, but instead backing by a customizable vtable.

- Removed ability to mutate the underlying memory from the `Bytes` type.
- Removed the "inline" (SBO) mechanism in `Bytes`. The reduces a large
  amount of complexity, and improves performance when accessing the
  slice of bytes, since a branch is no longer needed to check if the
  data is inline.
- Removed `Bytes` knowledge of `BytesMut` (`BytesMut` may grow that
  knowledge back at a future point.)

Cargo.toml

@@ -28,4 +28,5 @@ serde = { version = "1.0", optional = true }
 either = { version = "1.5", default-features = false, optional = true }
 
 [dev-dependencies]
+loom = "0.2.8"
 serde_test = "1.0"

benches/bytes.rs

@@ -4,47 +4,11 @@
 extern crate test;
 
 use test::Bencher;
-use bytes::{Bytes, BytesMut, BufMut};
-
-#[bench]
-fn alloc_small(b: &mut Bencher) {
-    b.iter(|| {
-        for _ in 0..1024 {
-            test::black_box(BytesMut::with_capacity(12));
-        }
-    })
-}
-
-#[bench]
-fn alloc_mid(b: &mut Bencher) {
-    b.iter(|| {
-        test::black_box(BytesMut::with_capacity(128));
-    })
-}
-
-#[bench]
-fn alloc_big(b: &mut Bencher) {
-    b.iter(|| {
-        test::black_box(BytesMut::with_capacity(4096));
-    })
-}
-
-#[bench]
-fn split_off_and_drop(b: &mut Bencher) {
-    b.iter(|| {
-        for _ in 0..1024 {
-            let v = vec![10; 200];
-            let mut b = Bytes::from(v);
-            test::black_box(b.split_off(100));
-            test::black_box(b);
-        }
-    })
-}
+use bytes::Bytes;
 
 #[bench]
 fn deref_unique(b: &mut Bencher) {
-    let mut buf = BytesMut::with_capacity(4096);
-    buf.put(&[0u8; 1024][..]);
+    let buf = Bytes::from(vec![0; 1024]);
 
     b.iter(|| {
         for _ in 0..1024 {
@@ -53,30 +17,10 @@ fn deref_unique(b: &mut Bencher) {
     })
 }
 
-#[bench]
-fn deref_unique_unroll(b: &mut Bencher) {
-    let mut buf = BytesMut::with_capacity(4096);
-    buf.put(&[0u8; 1024][..]);
-
-    b.iter(|| {
-        for _ in 0..128 {
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-            test::black_box(&buf[..]);
-        }
-    })
-}
-
 #[bench]
 fn deref_shared(b: &mut Bencher) {
-    let mut buf = BytesMut::with_capacity(4096);
-    buf.put(&[0u8; 1024][..]);
-    let _b2 = buf.split_off(1024);
+    let buf = Bytes::from(vec![0; 1024]);
+    let _b2 = buf.clone();
 
     b.iter(|| {
         for _ in 0..1024 {
@@ -86,9 +30,8 @@ fn deref_shared(b: &mut Bencher) {
 }
 
 #[bench]
-fn deref_inline(b: &mut Bencher) {
-    let mut buf = BytesMut::with_capacity(8);
-    buf.put(&[0u8; 8][..]);
+fn deref_static(b: &mut Bencher) {
+    let buf = Bytes::from_static(b"hello world");
 
     b.iter(|| {
         for _ in 0..1024 {
@@ -97,33 +40,6 @@ fn deref_inline(b: &mut Bencher) {
     })
 }
 
-#[bench]
-fn deref_two(b: &mut Bencher) {
-    let mut buf1 = BytesMut::with_capacity(8);
-    buf1.put(&[0u8; 8][..]);
-
-    let mut buf2 = BytesMut::with_capacity(4096);
-    buf2.put(&[0u8; 1024][..]);
-
-    b.iter(|| {
-        for _ in 0..512 {
-            test::black_box(&buf1[..]);
-            test::black_box(&buf2[..]);
-        }
-    })
-}
-
-#[bench]
-fn clone_inline(b: &mut Bencher) {
-    let bytes = Bytes::from_static(b"hello world");
-
-    b.iter(|| {
-        for _ in 0..1024 {
-            test::black_box(&bytes.clone());
-        }
-    })
-}
-
 #[bench]
 fn clone_static(b: &mut Bencher) {
     let bytes = Bytes::from_static("hello world 1234567890 and have a good byte 0987654321".as_bytes());
@@ -136,8 +52,8 @@ fn clone_static(b: &mut Bencher) {
 }
 
 #[bench]
-fn clone_arc(b: &mut Bencher) {
-    let bytes = Bytes::from("hello world 1234567890 and have a good byte 0987654321".as_bytes());
+fn clone_shared(b: &mut Bencher) {
+    let bytes = Bytes::from(b"hello world 1234567890 and have a good byte 0987654321".to_vec());
 
     b.iter(|| {
         for _ in 0..1024 {
@@ -147,42 +63,14 @@ fn clone_arc(b: &mut Bencher) {
 }
 
 #[bench]
-fn alloc_write_split_to_mid(b: &mut Bencher) {
-    b.iter(|| {
-        let mut buf = BytesMut::with_capacity(128);
-        buf.put_slice(&[0u8; 64]);
-        test::black_box(buf.split_to(64));
-    })
-}
-
-#[bench]
-fn drain_write_drain(b: &mut Bencher) {
-    let data = [0u8; 128];
+fn clone_arc_vec(b: &mut Bencher) {
+    use std::sync::Arc;
+    let bytes = Arc::new(b"hello world 1234567890 and have a good byte 0987654321".to_vec());
 
     b.iter(|| {
-        let mut buf = BytesMut::with_capacity(1024);
-        let mut parts = Vec::with_capacity(8);
-
-        for _ in 0..8 {
-            buf.put(&data[..]);
-            parts.push(buf.split_to(128));
+        for _ in 0..1024 {
+            test::black_box(&bytes.clone());
         }
-
-        test::black_box(parts);
-    })
-}
-
-#[bench]
-fn fmt_write(b: &mut Bencher) {
-    use std::fmt::Write;
-    let mut buf = BytesMut::with_capacity(128);
-    let s = "foo bar baz quux lorem ipsum dolor et";
-
-    b.bytes = s.len() as u64;
-    b.iter(|| {
-        let _ = write!(buf, "{}", s);
-        test::black_box(&buf);
-        unsafe { buf.set_len(0); }
     })
 }
 
@@ -191,7 +79,7 @@ fn from_long_slice(b: &mut Bencher) {
     let data = [0u8; 128];
     b.bytes = data.len() as u64;
     b.iter(|| {
-        let buf = BytesMut::from(&data[..]);
+        let buf = Bytes::copy_from_slice(&data[..]);
         test::black_box(buf);
     })
 }
@@ -248,3 +136,15 @@ fn slice_large_le_inline_from_arc(b: &mut Bencher) {
         }
     })
 }
+
+#[bench]
+fn split_off_and_drop(b: &mut Bencher) {
+    b.iter(|| {
+        for _ in 0..1024 {
+            let v = vec![10; 200];
+            let mut b = Bytes::from(v);
+            test::black_box(b.split_off(100));
+            test::black_box(b);
+        }
+    })
+}

benches/bytes_mut.rs

@@ -0,0 +1,142 @@
+#![feature(test)]
+#![deny(warnings, rust_2018_idioms)]
+
+extern crate test;
+
+use test::Bencher;
+use bytes::{BufMut, BytesMut};
+
+#[bench]
+fn alloc_small(b: &mut Bencher) {
+    b.iter(|| {
+        for _ in 0..1024 {
+            test::black_box(BytesMut::with_capacity(12));
+        }
+    })
+}
+
+#[bench]
+fn alloc_mid(b: &mut Bencher) {
+    b.iter(|| {
+        test::black_box(BytesMut::with_capacity(128));
+    })
+}
+
+#[bench]
+fn alloc_big(b: &mut Bencher) {
+    b.iter(|| {
+        test::black_box(BytesMut::with_capacity(4096));
+    })
+}
+
+
+#[bench]
+fn deref_unique(b: &mut Bencher) {
+    let mut buf = BytesMut::with_capacity(4096);
+    buf.put(&[0u8; 1024][..]);
+
+    b.iter(|| {
+        for _ in 0..1024 {
+            test::black_box(&buf[..]);
+        }
+    })
+}
+
+#[bench]
+fn deref_unique_unroll(b: &mut Bencher) {
+    let mut buf = BytesMut::with_capacity(4096);
+    buf.put(&[0u8; 1024][..]);
+
+    b.iter(|| {
+        for _ in 0..128 {
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+            test::black_box(&buf[..]);
+        }
+    })
+}
+
+#[bench]
+fn deref_shared(b: &mut Bencher) {
+    let mut buf = BytesMut::with_capacity(4096);
+    buf.put(&[0u8; 1024][..]);
+    let _b2 = buf.split_off(1024);
+
+    b.iter(|| {
+        for _ in 0..1024 {
+            test::black_box(&buf[..]);
+        }
+    })
+}
+
+#[bench]
+fn deref_two(b: &mut Bencher) {
+    let mut buf1 = BytesMut::with_capacity(8);
+    buf1.put(&[0u8; 8][..]);
+
+    let mut buf2 = BytesMut::with_capacity(4096);
+    buf2.put(&[0u8; 1024][..]);
+
+    b.iter(|| {
+        for _ in 0..512 {
+            test::black_box(&buf1[..]);
+            test::black_box(&buf2[..]);
+        }
+    })
+}
+
+#[bench]
+fn clone_frozen(b: &mut Bencher) {
+    let bytes = BytesMut::from(&b"hello world 1234567890 and have a good byte 0987654321"[..]).split().freeze();
+
+    b.iter(|| {
+        for _ in 0..1024 {
+            test::black_box(&bytes.clone());
+        }
+    })
+}
+
+#[bench]
+fn alloc_write_split_to_mid(b: &mut Bencher) {
+    b.iter(|| {
+        let mut buf = BytesMut::with_capacity(128);
+        buf.put_slice(&[0u8; 64]);
+        test::black_box(buf.split_to(64));
+    })
+}
+
+#[bench]
+fn drain_write_drain(b: &mut Bencher) {
+    let data = [0u8; 128];
+
+    b.iter(|| {
+        let mut buf = BytesMut::with_capacity(1024);
+        let mut parts = Vec::with_capacity(8);
+
+        for _ in 0..8 {
+            buf.put(&data[..]);
+            parts.push(buf.split_to(128));
+        }
+
+        test::black_box(parts);
+    })
+}
+
+#[bench]
+fn fmt_write(b: &mut Bencher) {
+    use std::fmt::Write;
+    let mut buf = BytesMut::with_capacity(128);
+    let s = "foo bar baz quux lorem ipsum dolor et";
+
+    b.bytes = s.len() as u64;
+    b.iter(|| {
+        let _ = write!(buf, "{}", s);
+        test::black_box(&buf);
+        unsafe { buf.set_len(0); }
+    })
+}

ci/azure-cross-compile.yml

@@ -1,5 +1,5 @@
 parameters:
-  cmd: test
+  cmd: build
   rust_version: stable
 
 jobs:

ci/azure-tsan.yml

@@ -18,9 +18,9 @@ jobs:
 
       # Run address sanitizer
       RUSTFLAGS="-Z sanitizer=address" \
-      cargo test --tests --target x86_64-unknown-linux-gnu
+      cargo test --target x86_64-unknown-linux-gnu --test test_bytes --test test_buf --test test_buf_mut
 
       # Run thread sanitizer
       RUSTFLAGS="-Z sanitizer=thread" \
-      cargo test --tests --target x86_64-unknown-linux-gnu
+      cargo test --target x86_64-unknown-linux-gnu --test test_bytes --test test_buf --test test_buf_mut
     displayName: TSAN / MSAN

ci/tsan

@@ -19,10 +19,6 @@ race:test::run_tests_console::*closure
 # Probably more fences in std.
 race:__call_tls_dtors
 
-# `is_inline_or_static` is explicitly called concurrently without synchronization.
-# The safety explanation can be found in a comment.
-race:Inner::is_inline_or_static
-
 # This ignores a false positive caused by `thread::park()`/`thread::unpark()`.
 # See: https://github.com/rust-lang/rust/pull/54806#issuecomment-436193353
 race:pthread_cond_destroy