Make CipherCtx::cipher_update more flexible

This change relaxes constraints on the output buffer size when it can be
safely determined how many bytes will be put in the output buffer.

For supported cryptographic backends (OpenSSL >= 1.1) the cipher's `num`
parameter will be consulted for the number of bytes in the block cache.
For unsupported backends the behavior will not change (the code will
assume full block in the cache).

For callers that do the check themselves and want to use other backends
(e.g. BoringSSL or LibreSSL) and unsafe `cipher_update_unchecked`
function is added.

Additionally a `CipherCtx::minimal_output_size` function is added for
letting the callers know how big should the output buffer be for the
next `cipher_update` call.

Fixes #1729.

See: https://mta.openssl.org/pipermail/openssl-users/2022-November/015623.html

openssl/src/cipher_ctx.rs

@@ -379,6 +379,49 @@ impl CipherCtxRef {
         unsafe { ffi::EVP_CIPHER_CTX_num(self.as_ptr()) as usize }
     }
 
+    /// Returns number of bytes cached in partial block update.
+    #[cfg(ossl110)]
+    fn used_block_size(&self) -> usize {
+        self.num()
+    }
+
+    /// Returns maximum number of bytes that could be cached.
+    #[cfg(not(ossl110))]
+    fn used_block_size(&self) -> usize {
+        self.block_size()
+    }
+
+    /// Calculate the minimal size of the output buffer given the
+    /// input buffer size.
+    ///
+    /// For streaming ciphers the minimal output size is the same as
+    /// the input size. For block ciphers the minimal output size
+    /// additionally depends on the partial blocks that might have
+    /// been written in previous calls to [`Self::cipher_update`].
+    ///
+    /// This function takes into account the number of partially
+    /// written blocks for block ciphers for supported targets
+    /// (OpenSSL >= 1.1). For unsupported targets the number of
+    /// partially written bytes is assumed to contain one full block
+    /// (pessimistic case).
+    ///
+    /// # Panics
+    ///
+    /// Panics if the context has not been initialized with a cipher.
+    pub fn minimal_output_size(&self, inlen: usize) -> usize {
+        let block_size = self.block_size();
+        if block_size > 1 {
+            // block cipher
+            let num = self.used_block_size();
+            let total_size = inlen + num;
+            let num_blocks = total_size / block_size;
+            num_blocks * block_size
+        } else {
+            // streaming cipher
+            inlen
+        }
+    }
+
     /// Sets the length of the IV expected by this context.
     ///
     /// Only some ciphers support configurable IV lengths.
@@ -517,33 +560,61 @@ impl CipherCtxRef {
     ///
     /// # Panics
     ///
-    /// Panics if `output.len()` is less than `input.len()` plus the cipher's block size.
+    /// Panics if `output` doesn't contain enough space for data to be
+    /// written as specified by [`Self::minimal_output_size`].
     #[corresponds(EVP_CipherUpdate)]
     pub fn cipher_update(
         &mut self,
         input: &[u8],
         output: Option<&mut [u8]>,
     ) -> Result<usize, ErrorStack> {
-        let inlen = c_int::try_from(input.len()).unwrap();
-
         if let Some(output) = &output {
-            let mut block_size = self.block_size();
-            if block_size == 1 {
-                block_size = 0;
-            }
-            assert!(output.len() >= input.len() + block_size);
+            let min_output_size = self.minimal_output_size(input.len());
+            assert!(
+                output.len() >= min_output_size,
+                "Output buffer size should be at least {} bytes.",
+                min_output_size
+            );
         }
 
+        unsafe { self.cipher_update_unchecked(input, output) }
+    }
+
+    /// Writes data into the context.
+    ///
+    /// Providing no output buffer will cause the input to be considered additional authenticated data (AAD).
+    ///
+    /// Returns the number of bytes written to `output`.
+    ///
+    /// This function is the same as [`Self::cipher_update`] but with the
+    /// output size check removed. It can be used when the exact
+    /// buffer size control is maintained by the caller and the
+    /// underlying cryptographic library doesn't expose exact block
+    /// cache data (e.g. OpenSSL < 1.1, BoringSSL, LibreSSL).
+    ///
+    /// SAFETY: The caller is expected to provide `output` buffer
+    /// large enough to contain correct number of bytes. For streaming
+    /// ciphers the output buffer size should be at least as big as
+    /// the input buffer. For block ciphers the size of the output
+    /// buffer depends on the state of partially updated blocks (see
+    /// [`Self::minimal_output_size`]).
+    #[corresponds(EVP_CipherUpdate)]
+    pub unsafe fn cipher_update_unchecked(
+        &mut self,
+        input: &[u8],
+        output: Option<&mut [u8]>,
+    ) -> Result<usize, ErrorStack> {
+        let inlen = c_int::try_from(input.len()).unwrap();
+
         let mut outlen = 0;
-        unsafe {
-            cvt(ffi::EVP_CipherUpdate(
-                self.as_ptr(),
-                output.map_or(ptr::null_mut(), |b| b.as_mut_ptr()),
-                &mut outlen,
-                input.as_ptr(),
-                inlen,
-            ))?;
-        }
+
+        cvt(ffi::EVP_CipherUpdate(
+            self.as_ptr(),
+            output.map_or(ptr::null_mut(), |b| b.as_mut_ptr()),
+            &mut outlen,
+            input.as_ptr(),
+            inlen,
+        ))?;
 
         Ok(outlen as usize)
     }
@@ -604,7 +675,7 @@ impl CipherCtxRef {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::cipher::Cipher;
+    use crate::{cipher::Cipher, rand::rand_bytes};
     #[cfg(not(boringssl))]
     use std::slice;
 
@@ -685,4 +756,201 @@ mod test {
         let cipher = Cipher::aes_128_cbc();
         aes_128_cbc(cipher);
     }
+
+    #[test]
+    #[cfg(ossl110)]
+    fn partial_block_updates() {
+        test_block_cipher_for_partial_block_updates(Cipher::aes_128_cbc());
+        test_block_cipher_for_partial_block_updates(Cipher::aes_256_cbc());
+        test_block_cipher_for_partial_block_updates(Cipher::des_ede3_cbc());
+    }
+
+    #[cfg(ossl110)]
+    fn test_block_cipher_for_partial_block_updates(cipher: &'static CipherRef) {
+        let mut key = vec![0; cipher.key_length()];
+        rand_bytes(&mut key).unwrap();
+        let mut iv = vec![0; cipher.iv_length()];
+        rand_bytes(&mut iv).unwrap();
+
+        let mut ctx = CipherCtx::new().unwrap();
+
+        ctx.encrypt_init(Some(cipher), Some(&key), Some(&iv))
+            .unwrap();
+        ctx.set_padding(false);
+
+        let block_size = cipher.block_size();
+        assert!(block_size > 1, "Need a block cipher, not a stream cipher");
+
+        // update cipher with non-full block
+        // expect no output until a block is complete
+        let outlen = ctx
+            .cipher_update(&vec![0; block_size - 1], Some(&mut [0; 0]))
+            .unwrap();
+        assert_eq!(0, outlen);
+
+        // update cipher with missing bytes from the previous block
+        // and one additional block, output should contain two blocks
+        let mut two_blocks = vec![0; block_size * 2];
+        let outlen = ctx
+            .cipher_update(&vec![0; block_size + 1], Some(&mut two_blocks))
+            .unwrap();
+        assert_eq!(block_size * 2, outlen);
+
+        ctx.cipher_final_vec(&mut vec![0; 0]).unwrap();
+
+        // try to decrypt
+        ctx.decrypt_init(Some(cipher), Some(&key), Some(&iv))
+            .unwrap();
+        ctx.set_padding(false);
+
+        // update cipher with non-full block
+        // expect no output until a block is complete
+        let outlen = ctx
+            .cipher_update(&two_blocks[0..block_size - 1], Some(&mut [0; 0]))
+            .unwrap();
+        assert_eq!(0, outlen);
+
+        // update cipher with missing bytes from the previous block
+        // and one additional block, output should contain two blocks
+        let mut two_blocks_decrypted = vec![0; block_size * 2];
+        let outlen = ctx
+            .cipher_update(
+                &two_blocks[block_size - 1..],
+                Some(&mut two_blocks_decrypted),
+            )
+            .unwrap();
+        assert_eq!(block_size * 2, outlen);
+
+        ctx.cipher_final_vec(&mut vec![0; 0]).unwrap();
+        // check if the decrypted blocks are the same as input (all zeros)
+        assert_eq!(two_blocks_decrypted, vec![0; block_size * 2]);
+    }
+
+    #[test]
+    fn test_stream_ciphers() {
+        test_stream_cipher(Cipher::aes_192_ctr());
+        test_stream_cipher(Cipher::aes_256_ctr());
+    }
+
+    fn test_stream_cipher(cipher: &'static CipherRef) {
+        let mut key = vec![0; cipher.key_length()];
+        rand_bytes(&mut key).unwrap();
+        let mut iv = vec![0; cipher.iv_length()];
+        rand_bytes(&mut iv).unwrap();
+
+        let mut ctx = CipherCtx::new().unwrap();
+
+        ctx.encrypt_init(Some(cipher), Some(&key), Some(&iv))
+            .unwrap();
+        ctx.set_padding(false);
+
+        assert_eq!(
+            1,
+            cipher.block_size(),
+            "Need a stream cipher, not a block cipher"
+        );
+
+        // update cipher with non-full block
+        // this is a streaming cipher so the number of output bytes
+        // will be the same as the number of input bytes
+        let mut output = vec![0; 32];
+        let outlen = ctx
+            .cipher_update(&[1; 15], Some(&mut output[0..15]))
+            .unwrap();
+        assert_eq!(15, outlen);
+
+        // update cipher with missing bytes from the previous block
+        // as previously it will output the same number of bytes as
+        // the input
+        let outlen = ctx
+            .cipher_update(&[1; 17], Some(&mut output[15..]))
+            .unwrap();
+        assert_eq!(17, outlen);
+
+        ctx.cipher_final_vec(&mut vec![0; 0]).unwrap();
+
+        // try to decrypt
+        ctx.decrypt_init(Some(cipher), Some(&key), Some(&iv))
+            .unwrap();
+        ctx.set_padding(false);
+
+        // update cipher with non-full block
+        // expect that the output for stream cipher will contain
+        // the same number of bytes as the input
+        let mut output_decrypted = vec![0; 32];
+        let outlen = ctx
+            .cipher_update(&output[0..15], Some(&mut output_decrypted[0..15]))
+            .unwrap();
+        assert_eq!(15, outlen);
+
+        let outlen = ctx
+            .cipher_update(&output[15..], Some(&mut output_decrypted[15..]))
+            .unwrap();
+        assert_eq!(17, outlen);
+
+        ctx.cipher_final_vec(&mut vec![0; 0]).unwrap();
+        // check if the decrypted blocks are the same as input (all ones)
+        assert_eq!(output_decrypted, vec![1; 32]);
+    }
+
+    #[test]
+    #[should_panic(expected = "Output buffer size should be at least 16 bytes.")]
+    #[cfg(ossl110)]
+    fn full_block_updates_aes_128() {
+        output_buffer_too_small(Cipher::aes_128_cbc());
+    }
+
+    #[test]
+    #[should_panic(expected = "Output buffer size should be at least 16 bytes.")]
+    #[cfg(ossl110)]
+    fn full_block_updates_aes_256() {
+        output_buffer_too_small(Cipher::aes_256_cbc());
+    }
+
+    #[test]
+    #[should_panic(expected = "Output buffer size should be at least 8 bytes.")]
+    #[cfg(ossl110)]
+    fn full_block_updates_3des() {
+        output_buffer_too_small(Cipher::des_ede3_cbc());
+    }
+
+    #[test]
+    #[should_panic(expected = "Output buffer size should be at least 32 bytes.")]
+    #[cfg(not(ossl110))]
+    fn full_block_updates_aes_128() {
+        output_buffer_too_small(Cipher::aes_128_cbc());
+    }
+
+    #[test]
+    #[should_panic(expected = "Output buffer size should be at least 32 bytes.")]
+    #[cfg(not(ossl110))]
+    fn full_block_updates_aes_256() {
+        output_buffer_too_small(Cipher::aes_256_cbc());
+    }
+
+    #[test]
+    #[should_panic(expected = "Output buffer size should be at least 16 bytes.")]
+    #[cfg(not(ossl110))]
+    fn full_block_updates_3des() {
+        output_buffer_too_small(Cipher::des_ede3_cbc());
+    }
+
+    fn output_buffer_too_small(cipher: &'static CipherRef) {
+        let mut key = vec![0; cipher.key_length()];
+        rand_bytes(&mut key).unwrap();
+        let mut iv = vec![0; cipher.iv_length()];
+        rand_bytes(&mut iv).unwrap();
+
+        let mut ctx = CipherCtx::new().unwrap();
+
+        ctx.encrypt_init(Some(cipher), Some(&key), Some(&iv))
+            .unwrap();
+        ctx.set_padding(false);
+
+        let block_size = cipher.block_size();
+        assert!(block_size > 1, "Need a block cipher, not a stream cipher");
+
+        ctx.cipher_update(&vec![0; block_size + 1], Some(&mut vec![0; block_size - 1]))
+            .unwrap();
+    }
 }