From f31dd79edf773697d4823682c3ec9d5f10397189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Fri, 24 Jul 2026 17:57:06 +0300 Subject: [PATCH 1/3] aes: refactor backends --- Cargo.lock | 8 +- aes/CHANGELOG.md | 10 + aes/Cargo.toml | 2 +- aes/src/armv8.rs | 321 --- aes/src/armv8/test_expand.rs | 146 -- aes/src/autodetect.rs | 403 ---- aes/src/backends.rs | 24 + aes/src/backends/aarch64_aes.rs | 223 ++ .../{armv8 => backends/aarch64_aes}/encdec.rs | 4 + .../{armv8 => backends/aarch64_aes}/expand.rs | 36 +- .../{armv8 => backends/aarch64_aes}/hazmat.rs | 0 aes/src/backends/fixslice/aes128.rs | 167 ++ aes/src/backends/fixslice/aes192.rs | 203 ++ aes/src/backends/fixslice/aes256.rs | 175 ++ aes/src/backends/fixslice/hazmat.rs | 138 ++ aes/src/backends/fixslice/mix_columns.rs | 174 ++ aes/src/backends/fixslice/mod.rs | 43 + aes/src/backends/fixslice/sbox.rs | 395 ++++ aes/src/backends/fixslice/utils.rs | 119 ++ aes/src/backends/fixslice/word.rs | 324 +++ aes/src/backends/soft.rs | 97 + aes/src/backends/x86_aes.rs | 228 ++ aes/src/{ni => backends/x86_aes}/encdec.rs | 89 +- aes/src/{ni => backends/x86_aes}/expand.rs | 69 +- .../{x86/ni => backends/x86_aes}/hazmat.rs | 6 +- aes/src/backends/x86_vaes256.rs | 89 + aes/src/backends/x86_vaes256/encdec.rs | 109 + aes/src/backends/x86_vaes512.rs | 91 + aes/src/backends/x86_vaes512/encdec.rs | 109 + aes/src/hazmat.rs | 28 +- aes/src/lib.rs | 427 +++- aes/src/macros.rs | 105 - aes/src/ni.rs | 339 --- aes/src/ni/hazmat.rs | 97 - aes/src/ni/test_expand.rs | 290 --- aes/src/soft.rs | 309 --- aes/src/soft/fixslice.rs | 1842 ----------------- aes/src/x86.rs | 712 ------- aes/src/x86/ni.rs | 24 - aes/src/x86/ni/encdec.rs | 161 -- aes/src/x86/ni/expand.rs | 219 -- aes/src/x86/ni/test_expand.rs | 291 --- aes/src/x86/vaes256.rs | 1 - aes/src/x86/vaes256/encdec.rs | 83 - aes/src/x86/vaes512.rs | 1 - aes/src/x86/vaes512/encdec.rs | 85 - 46 files changed, 3245 insertions(+), 5571 deletions(-) delete mode 100644 aes/src/armv8.rs delete mode 100644 aes/src/armv8/test_expand.rs delete mode 100644 aes/src/autodetect.rs create mode 100644 aes/src/backends.rs create mode 100644 aes/src/backends/aarch64_aes.rs rename aes/src/{armv8 => backends/aarch64_aes}/encdec.rs (99%) rename aes/src/{armv8 => backends/aarch64_aes}/expand.rs (69%) rename aes/src/{armv8 => backends/aarch64_aes}/hazmat.rs (100%) create mode 100644 aes/src/backends/fixslice/aes128.rs create mode 100644 aes/src/backends/fixslice/aes192.rs create mode 100644 aes/src/backends/fixslice/aes256.rs create mode 100644 aes/src/backends/fixslice/hazmat.rs create mode 100644 aes/src/backends/fixslice/mix_columns.rs create mode 100644 aes/src/backends/fixslice/mod.rs create mode 100644 aes/src/backends/fixslice/sbox.rs create mode 100644 aes/src/backends/fixslice/utils.rs create mode 100644 aes/src/backends/fixslice/word.rs create mode 100644 aes/src/backends/soft.rs create mode 100644 aes/src/backends/x86_aes.rs rename aes/src/{ni => backends/x86_aes}/encdec.rs (61%) rename aes/src/{ni => backends/x86_aes}/expand.rs (71%) rename aes/src/{x86/ni => backends/x86_aes}/hazmat.rs (96%) create mode 100644 aes/src/backends/x86_vaes256.rs create mode 100644 aes/src/backends/x86_vaes256/encdec.rs create mode 100644 aes/src/backends/x86_vaes512.rs create mode 100644 aes/src/backends/x86_vaes512/encdec.rs delete mode 100644 aes/src/macros.rs delete mode 100644 aes/src/ni.rs delete mode 100644 aes/src/ni/hazmat.rs delete mode 100644 aes/src/ni/test_expand.rs delete mode 100644 aes/src/soft.rs delete mode 100644 aes/src/soft/fixslice.rs delete mode 100644 aes/src/x86.rs delete mode 100644 aes/src/x86/ni.rs delete mode 100644 aes/src/x86/ni/encdec.rs delete mode 100644 aes/src/x86/ni/expand.rs delete mode 100644 aes/src/x86/ni/test_expand.rs delete mode 100644 aes/src/x86/vaes256.rs delete mode 100644 aes/src/x86/vaes256/encdec.rs delete mode 100644 aes/src/x86/vaes512.rs delete mode 100644 aes/src/x86/vaes512/encdec.rs diff --git a/Cargo.lock b/Cargo.lock index d58529b2f..743cfd399 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,9 +149,9 @@ checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "typenum", "zeroize", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "magma" diff --git a/aes/CHANGELOG.md b/aes/CHANGELOG.md index f40039d1e..4ce2bd653 100644 --- a/aes/CHANGELOG.md +++ b/aes/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.9.2 (UNRELEASED) +### Changed +- Internal implementation of backends ([#560], [#575]) + +### Fixed +- Performance regression on x86 targets ([#575]) + +[#560]: https://github.com/RustCrypto/block-ciphers/pull/560 +[#575]: https://github.com/RustCrypto/block-ciphers/pull/575 + ## 0.9.1 (2026-05-27) ### Fixed - Minimal version of `zeroize` dependency ([#562]) diff --git a/aes/Cargo.toml b/aes/Cargo.toml index db53df7a0..99a49dd6b 100644 --- a/aes/Cargo.toml +++ b/aes/Cargo.toml @@ -15,7 +15,7 @@ categories = ["cryptography", "no-std"] [dependencies] cipher = "0.5" cpubits = "0.1" -zeroize = { version = "1.8", optional = true, default-features = false, features = ["aarch64"] } +zeroize = { version = "1.9", optional = true, default-features = false } [target.'cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))'.dependencies] cpufeatures = "0.3" diff --git a/aes/src/armv8.rs b/aes/src/armv8.rs deleted file mode 100644 index aed6caf29..000000000 --- a/aes/src/armv8.rs +++ /dev/null @@ -1,321 +0,0 @@ -//! AES block cipher implementation using the ARMv8 Cryptography Extensions. -//! -//! Based on this C intrinsics implementation: -//! -//! -//! Original C written and placed in public domain by Jeffrey Walton. -//! Based on code from ARM, and by Johannes Schneiders, Skip Hovsmith and -//! Barry O'Rourke for the mbedTLS project. - -#![allow(clippy::needless_range_loop)] - -#[cfg(feature = "hazmat")] -pub(crate) mod hazmat; - -mod encdec; -mod expand; -#[cfg(test)] -mod test_expand; - -use cipher::{ - AlgorithmName, BlockCipherDecClosure, BlockCipherDecrypt, BlockCipherEncClosure, - BlockCipherEncrypt, BlockSizeUser, Key, KeyInit, KeySizeUser, - consts::{self, U16, U24, U32}, -}; -use core::fmt; - -pub(crate) mod features { - cpufeatures::new!(features_aes, "aes"); - pub(crate) mod aes { - pub use super::features_aes::*; - } -} - -impl_backends!( - enc_name = Aes128BackEnc, - dec_name = Aes128BackDec, - key_size = consts::U16, - keys_ty = expand::Aes128RoundKeys, - par_size = consts::U21, - expand_keys = expand::expand_key, - inv_keys = expand::inv_expanded_keys, - encrypt = encdec::encrypt, - encrypt_par = encdec::encrypt_par, - decrypt = encdec::decrypt, - decrypt_par = encdec::decrypt_par, -); - -impl_backends!( - enc_name = Aes192BackEnc, - dec_name = Aes192BackDec, - key_size = consts::U24, - keys_ty = expand::Aes192RoundKeys, - par_size = consts::U19, - expand_keys = expand::expand_key, - inv_keys = expand::inv_expanded_keys, - encrypt = encdec::encrypt, - encrypt_par = encdec::encrypt_par, - decrypt = encdec::decrypt, - decrypt_par = encdec::decrypt_par, -); - -impl_backends!( - enc_name = Aes256BackEnc, - dec_name = Aes256BackDec, - key_size = consts::U32, - keys_ty = expand::Aes256RoundKeys, - par_size = consts::U17, - expand_keys = expand::expand_key, - inv_keys = expand::inv_expanded_keys, - encrypt = encdec::encrypt, - encrypt_par = encdec::encrypt_par, - decrypt = encdec::decrypt, - decrypt_par = encdec::decrypt_par, -); - -macro_rules! define_aes_impl { - ( - $name:ident, - $name_enc:ident, - $name_dec:ident, - $name_back_enc:ident, - $name_back_dec:ident, - $key_size:ty, - $rounds:tt, - $doc:expr $(,)? - ) => { - #[doc=$doc] - #[doc = "block cipher"] - #[derive(Clone)] - pub struct $name { - encrypt: $name_back_enc, - decrypt: $name_back_dec, - } - - impl KeySizeUser for $name { - type KeySize = $key_size; - } - - impl KeyInit for $name { - #[inline] - fn new(key: &Key) -> Self { - let encrypt = $name_back_enc::new(key); - let decrypt = $name_back_dec::from(encrypt.clone()); - Self { encrypt, decrypt } - } - } - - impl From<$name_enc> for $name { - #[inline] - fn from(encrypt: $name_enc) -> $name { - let encrypt = encrypt.backend.clone(); - let decrypt = encrypt.clone().into(); - Self { encrypt, decrypt } - } - } - - impl From<&$name_enc> for $name { - #[inline] - fn from(encrypt: &$name_enc) -> $name { - let encrypt = encrypt.backend.clone(); - let decrypt = encrypt.clone().into(); - Self { encrypt, decrypt } - } - } - - impl BlockSizeUser for $name { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - f.call(&self.encrypt) - } - } - - impl BlockCipherDecrypt for $name { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - f.call(&self.decrypt) - } - } - - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name), " { .. }")) - } - } - - impl AlgorithmName for $name { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name)) - } - } - - impl Drop for $name { - #[inline] - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(self); - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name {} - - #[doc=$doc] - #[doc = "block cipher (encrypt-only)"] - #[derive(Clone)] - pub struct $name_enc { - backend: $name_back_enc, - } - - impl KeySizeUser for $name_enc { - type KeySize = $key_size; - } - - impl KeyInit for $name_enc { - #[inline] - fn new(key: &Key) -> Self { - let backend = $name_back_enc::new(key); - Self { backend } - } - } - - impl BlockSizeUser for $name_enc { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name_enc { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - f.call(&self.backend) - } - } - - impl fmt::Debug for $name_enc { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_enc), " { .. }")) - } - } - - impl AlgorithmName for $name_enc { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_enc)) - } - } - - impl Drop for $name_enc { - #[inline] - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(self); - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_enc {} - - #[doc=$doc] - #[doc = "block cipher (decrypt-only)"] - #[derive(Clone)] - pub struct $name_dec { - backend: $name_back_dec, - } - - impl KeySizeUser for $name_dec { - type KeySize = $key_size; - } - - impl KeyInit for $name_dec { - #[inline] - fn new(key: &Key) -> Self { - let encrypt = $name_back_enc::new(key); - let backend = encrypt.clone().into(); - Self { backend } - } - } - - impl From<$name_enc> for $name_dec { - #[inline] - fn from(enc: $name_enc) -> $name_dec { - Self::from(&enc) - } - } - - impl From<&$name_enc> for $name_dec { - fn from(encrypt: &$name_enc) -> $name_dec { - let backend = encrypt.backend.clone().into(); - Self { backend } - } - } - - impl BlockSizeUser for $name_dec { - type BlockSize = U16; - } - - impl BlockCipherDecrypt for $name_dec { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - f.call(&self.backend); - } - } - - impl fmt::Debug for $name_dec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_dec), " { .. }")) - } - } - - impl AlgorithmName for $name_dec { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_dec)) - } - } - - impl Drop for $name_dec { - #[inline] - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(self); - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_dec {} - }; -} - -define_aes_impl!( - Aes128, - Aes128Enc, - Aes128Dec, - Aes128BackEnc, - Aes128BackDec, - U16, - 11, - "AES-128", -); -define_aes_impl!( - Aes192, - Aes192Enc, - Aes192Dec, - Aes192BackEnc, - Aes192BackDec, - U24, - 13, - "AES-192", -); -define_aes_impl!( - Aes256, - Aes256Enc, - Aes256Dec, - Aes256BackEnc, - Aes256BackDec, - U32, - 15, - "AES-256", -); diff --git a/aes/src/armv8/test_expand.rs b/aes/src/armv8/test_expand.rs deleted file mode 100644 index ad41aa7ad..000000000 --- a/aes/src/armv8/test_expand.rs +++ /dev/null @@ -1,146 +0,0 @@ -use super::expand::{expand_key, inv_expanded_keys}; -use core::arch::aarch64::*; -use hex_literal::hex; - -/// FIPS 197, Appendix A.1: AES-128 Cipher Key -/// user input, unaligned buffer -const AES128_KEY: [u8; 16] = hex!("2b7e151628aed2a6abf7158809cf4f3c"); - -/// FIPS 197 Appendix A.1: Expansion of a 128-bit Cipher Key -/// library controlled, aligned buffer -const AES128_EXP_KEYS: [[u8; 16]; 11] = [ - AES128_KEY, - hex!("a0fafe1788542cb123a339392a6c7605"), - hex!("f2c295f27a96b9435935807a7359f67f"), - hex!("3d80477d4716fe3e1e237e446d7a883b"), - hex!("ef44a541a8525b7fb671253bdb0bad00"), - hex!("d4d1c6f87c839d87caf2b8bc11f915bc"), - hex!("6d88a37a110b3efddbf98641ca0093fd"), - hex!("4e54f70e5f5fc9f384a64fb24ea6dc4f"), - hex!("ead27321b58dbad2312bf5607f8d292f"), - hex!("ac7766f319fadc2128d12941575c006e"), - hex!("d014f9a8c9ee2589e13f0cc8b6630ca6"), -]; - -/// Inverse expanded keys for [`AES128_EXPANDED_KEYS`] -const AES128_EXP_INVKEYS: [[u8; 16]; 11] = [ - hex!("d014f9a8c9ee2589e13f0cc8b6630ca6"), - hex!("0c7b5a631319eafeb0398890664cfbb4"), - hex!("df7d925a1f62b09da320626ed6757324"), - hex!("12c07647c01f22c7bc42d2f37555114a"), - hex!("6efcd876d2df54807c5df034c917c3b9"), - hex!("6ea30afcbc238cf6ae82a4b4b54a338d"), - hex!("90884413d280860a12a128421bc89739"), - hex!("7c1f13f74208c219c021ae480969bf7b"), - hex!("cc7505eb3e17d1ee82296c51c9481133"), - hex!("2b3708a7f262d405bc3ebdbf4b617d62"), - AES128_KEY, -]; - -/// FIPS 197, Appendix A.2: AES-192 Cipher Key -/// user input, unaligned buffer -const AES192_KEY: [u8; 24] = hex!("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"); - -/// FIPS 197 Appendix A.2: Expansion of a 192-bit Cipher Key -/// library controlled, aligned buffer -const AES192_EXP_KEYS: [[u8; 16]; 13] = [ - hex!("8e73b0f7da0e6452c810f32b809079e5"), - hex!("62f8ead2522c6b7bfe0c91f72402f5a5"), - hex!("ec12068e6c827f6b0e7a95b95c56fec2"), - hex!("4db7b4bd69b5411885a74796e92538fd"), - hex!("e75fad44bb095386485af05721efb14f"), - hex!("a448f6d94d6dce24aa326360113b30e6"), - hex!("a25e7ed583b1cf9a27f939436a94f767"), - hex!("c0a69407d19da4e1ec1786eb6fa64971"), - hex!("485f703222cb8755e26d135233f0b7b3"), - hex!("40beeb282f18a2596747d26b458c553e"), - hex!("a7e1466c9411f1df821f750aad07d753"), - hex!("ca4005388fcc5006282d166abc3ce7b5"), - hex!("e98ba06f448c773c8ecc720401002202"), -]; - -/// FIPS 197, Appendix A.3: AES-256 Cipher Key -/// user input, unaligned buffer -const AES256_KEY: [u8; 32] = - hex!("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"); - -/// FIPS 197 Appendix A.3: Expansion of a 256-bit Cipher Key -/// library controlled, aligned buffer -const AES256_EXP_KEYS: [[u8; 16]; 15] = [ - hex!("603deb1015ca71be2b73aef0857d7781"), - hex!("1f352c073b6108d72d9810a30914dff4"), - hex!("9ba354118e6925afa51a8b5f2067fcde"), - hex!("a8b09c1a93d194cdbe49846eb75d5b9a"), - hex!("d59aecb85bf3c917fee94248de8ebe96"), - hex!("b5a9328a2678a647983122292f6c79b3"), - hex!("812c81addadf48ba24360af2fab8b464"), - hex!("98c5bfc9bebd198e268c3ba709e04214"), - hex!("68007bacb2df331696e939e46c518d80"), - hex!("c814e20476a9fb8a5025c02d59c58239"), - hex!("de1369676ccc5a71fa2563959674ee15"), - hex!("5886ca5d2e2f31d77e0af1fa27cf73c3"), - hex!("749c47ab18501ddae2757e4f7401905a"), - hex!("cafaaae3e4d59b349adf6acebd10190d"), - hex!("fe4890d1e6188d0b046df344706c631e"), -]; - -fn load_expanded_keys(input: [[u8; 16]; N]) -> [uint8x16_t; N] { - let mut output = [unsafe { vdupq_n_u8(0) }; N]; - - for (src, dst) in input.iter().zip(output.iter_mut()) { - *dst = unsafe { vld1q_u8(src.as_ptr()) } - } - - output -} - -fn store_expanded_keys(input: [uint8x16_t; N]) -> [[u8; 16]; N] { - let mut output = [[0u8; 16]; N]; - - for (src, dst) in input.iter().zip(output.iter_mut()) { - unsafe { vst1q_u8(dst.as_mut_ptr(), *src) } - } - - output -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes128_key_expansion() { - let ek = unsafe { expand_key(&AES128_KEY) }; - assert_eq!(store_expanded_keys(ek), AES128_EXP_KEYS); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes128_key_expansion_inv() { - let ek = load_expanded_keys(AES128_EXP_KEYS); - let inv_ek = unsafe { inv_expanded_keys(&ek) }; - assert_eq!(store_expanded_keys(inv_ek), AES128_EXP_INVKEYS); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes192_key_expansion() { - let ek = unsafe { expand_key(&AES192_KEY) }; - assert_eq!(store_expanded_keys(ek), AES192_EXP_KEYS); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes256_key_expansion() { - let ek = unsafe { expand_key(&AES256_KEY) }; - assert_eq!(store_expanded_keys(ek), AES256_EXP_KEYS); -} diff --git a/aes/src/autodetect.rs b/aes/src/autodetect.rs deleted file mode 100644 index 0591a4a43..000000000 --- a/aes/src/autodetect.rs +++ /dev/null @@ -1,403 +0,0 @@ -//! Autodetection support for hardware accelerated AES backends with fallback -//! to the fixsliced "soft" implementation. - -use crate::soft; -use cipher::{ - AlgorithmName, BlockCipherDecClosure, BlockCipherDecrypt, BlockCipherEncClosure, - BlockCipherEncrypt, BlockSizeUser, Key, KeyInit, KeySizeUser, - consts::{U16, U24, U32}, -}; -use core::fmt; -use core::mem::ManuallyDrop; - -#[cfg(target_arch = "aarch64")] -use crate::armv8 as arch; - -#[cfg(any(target_arch = "x86_64", target_arch = "x86"))] -use crate::x86 as arch; - -macro_rules! define_aes_impl { - ( - name = $name:ident, - name_enc = $name_enc:ident, - name_dec = $name_dec:ident, - module = $module:tt, - key_size = $key_size:ty, - doc = $doc:expr, - ) => { - mod $module { - use super::{arch, soft}; - use core::mem::ManuallyDrop; - - pub(super) union Inner { - pub(super) arch: ManuallyDrop, - pub(super) soft: ManuallyDrop, - } - - pub(super) union InnerEnc { - pub(super) arch: ManuallyDrop, - pub(super) soft: ManuallyDrop, - } - - pub(super) union InnerDec { - pub(super) arch: ManuallyDrop, - pub(super) soft: ManuallyDrop, - } - } - - #[doc=$doc] - #[doc = "block cipher"] - pub struct $name { - inner: $module::Inner, - token: arch::features::aes::InitToken, - } - - impl KeySizeUser for $name { - type KeySize = $key_size; - } - impl From<$name_enc> for $name { - #[inline] - fn from(enc: $name_enc) -> $name { - Self::from(&enc) - } - } - - impl From<&$name_enc> for $name { - fn from(enc: &$name_enc) -> $name { - use core::ops::Deref; - let inner = if enc.token.get() { - $module::Inner { - arch: ManuallyDrop::new(unsafe { enc.inner.arch.deref().into() }), - } - } else { - $module::Inner { - soft: ManuallyDrop::new(unsafe { enc.inner.soft.deref().into() }), - } - }; - - Self { - inner, - token: enc.token, - } - } - } - - impl KeyInit for $name { - #[inline] - fn new(key: &Key) -> Self { - let (token, aes_features) = arch::features::aes::init_get(); - - let inner = if aes_features { - $module::Inner { - arch: ManuallyDrop::new(arch::$name::new(key)), - } - } else { - $module::Inner { - soft: ManuallyDrop::new(soft::$name::new(key)), - } - }; - - Self { inner, token } - } - } - - impl Clone for $name { - fn clone(&self) -> Self { - let inner = if self.token.get() { - $module::Inner { - arch: unsafe { self.inner.arch.clone() }, - } - } else { - $module::Inner { - soft: unsafe { self.inner.soft.clone() }, - } - }; - - Self { - inner, - token: self.token, - } - } - } - - impl BlockSizeUser for $name { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - if self.token.get() { - unsafe { &self.inner.arch }.encrypt_with_backend(f) - } else { - unsafe { &self.inner.soft }.encrypt_with_backend(f) - } - } - } - - impl BlockCipherDecrypt for $name { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - if self.token.get() { - unsafe { &self.inner.arch }.decrypt_with_backend(f) - } else { - unsafe { &self.inner.soft }.decrypt_with_backend(f) - } - } - } - - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name), " { .. }")) - } - } - - impl AlgorithmName for $name { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name)) - } - } - - impl Drop for $name { - #[inline] - fn drop(&mut self) { - if self.token.get() { - unsafe { ManuallyDrop::drop(&mut self.inner.arch) }; - } else { - unsafe { ManuallyDrop::drop(&mut self.inner.soft) }; - }; - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name {} - - #[doc=$doc] - #[doc = "block cipher (encrypt-only)"] - pub struct $name_enc { - inner: $module::InnerEnc, - token: arch::features::aes::InitToken, - } - - impl KeySizeUser for $name_enc { - type KeySize = $key_size; - } - - impl KeyInit for $name_enc { - #[inline] - fn new(key: &Key) -> Self { - let (token, aes_features) = arch::features::aes::init_get(); - - let inner = if aes_features { - $module::InnerEnc { - arch: ManuallyDrop::new(arch::$name_enc::new(key)), - } - } else { - $module::InnerEnc { - soft: ManuallyDrop::new(soft::$name_enc::new(key)), - } - }; - - Self { inner, token } - } - } - - impl Clone for $name_enc { - fn clone(&self) -> Self { - let inner = if self.token.get() { - $module::InnerEnc { - arch: unsafe { self.inner.arch.clone() }, - } - } else { - $module::InnerEnc { - soft: unsafe { self.inner.soft.clone() }, - } - }; - - Self { - inner, - token: self.token, - } - } - } - - impl BlockSizeUser for $name_enc { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name_enc { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - if self.token.get() { - unsafe { &self.inner.arch }.encrypt_with_backend(f) - } else { - unsafe { &self.inner.soft }.encrypt_with_backend(f) - } - } - } - - impl fmt::Debug for $name_enc { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_enc), " { .. }")) - } - } - - impl AlgorithmName for $name_enc { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_enc)) - } - } - - impl Drop for $name_enc { - #[inline] - fn drop(&mut self) { - if self.token.get() { - unsafe { ManuallyDrop::drop(&mut self.inner.arch) }; - } else { - unsafe { ManuallyDrop::drop(&mut self.inner.soft) }; - }; - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_enc {} - - #[doc=$doc] - #[doc = "block cipher (decrypt-only)"] - pub struct $name_dec { - inner: $module::InnerDec, - token: arch::features::aes::InitToken, - } - - impl KeySizeUser for $name_dec { - type KeySize = $key_size; - } - - impl From<$name_enc> for $name_dec { - #[inline] - fn from(enc: $name_enc) -> $name_dec { - Self::from(&enc) - } - } - - impl From<&$name_enc> for $name_dec { - fn from(enc: &$name_enc) -> $name_dec { - use core::ops::Deref; - let inner = if enc.token.get() { - $module::InnerDec { - arch: ManuallyDrop::new(unsafe { enc.inner.arch.deref().into() }), - } - } else { - $module::InnerDec { - soft: ManuallyDrop::new(unsafe { enc.inner.soft.deref().into() }), - } - }; - - Self { - inner, - token: enc.token, - } - } - } - - impl KeyInit for $name_dec { - #[inline] - fn new(key: &Key) -> Self { - let (token, aes_features) = arch::features::aes::init_get(); - - let inner = if aes_features { - $module::InnerDec { - arch: ManuallyDrop::new(arch::$name_dec::new(key)), - } - } else { - $module::InnerDec { - soft: ManuallyDrop::new(soft::$name_dec::new(key)), - } - }; - - Self { inner, token } - } - } - - impl Clone for $name_dec { - fn clone(&self) -> Self { - let inner = if self.token.get() { - $module::InnerDec { - arch: unsafe { self.inner.arch.clone() }, - } - } else { - $module::InnerDec { - soft: unsafe { self.inner.soft.clone() }, - } - }; - - Self { - inner, - token: self.token, - } - } - } - - impl BlockSizeUser for $name_dec { - type BlockSize = U16; - } - - impl BlockCipherDecrypt for $name_dec { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - if self.token.get() { - unsafe { &self.inner.arch }.decrypt_with_backend(f) - } else { - unsafe { &self.inner.soft }.decrypt_with_backend(f) - } - } - } - - impl fmt::Debug for $name_dec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_dec), " { .. }")) - } - } - - impl AlgorithmName for $name_dec { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_dec)) - } - } - - impl Drop for $name_dec { - #[inline] - fn drop(&mut self) { - if self.token.get() { - unsafe { ManuallyDrop::drop(&mut self.inner.arch) }; - } else { - unsafe { ManuallyDrop::drop(&mut self.inner.soft) }; - }; - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_dec {} - }; -} - -define_aes_impl!( - name = Aes128, - name_enc = Aes128Enc, - name_dec = Aes128Dec, - module = aes128, - key_size = U16, - doc = "AES-128", -); -define_aes_impl!( - name = Aes192, - name_enc = Aes192Enc, - name_dec = Aes192Dec, - module = aes192, - key_size = U24, - doc = "AES-192", -); -define_aes_impl!( - name = Aes256, - name_enc = Aes256Enc, - name_dec = Aes256Dec, - module = aes256, - key_size = U32, - doc = "AES-256", -); diff --git a/aes/src/backends.rs b/aes/src/backends.rs new file mode 100644 index 000000000..59761d512 --- /dev/null +++ b/aes/src/backends.rs @@ -0,0 +1,24 @@ +pub(crate) mod soft; + +#[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] +pub(crate) mod aarch64_aes; + +#[cfg(all( + any(target_arch = "x86_64", target_arch = "x86"), + not(aes_backend = "soft") +))] +pub(crate) mod x86_aes; + +#[cfg(all( + any(aes_backend = "avx256", aes_backend = "avx512"), + any(target_arch = "x86_64", target_arch = "x86"), + not(aes_backend = "soft"), +))] +pub(crate) mod x86_vaes256; + +#[cfg(all( + aes_backend = "avx512", + any(target_arch = "x86_64", target_arch = "x86"), + not(aes_backend = "soft"), +))] +pub(crate) mod x86_vaes512; diff --git a/aes/src/backends/aarch64_aes.rs b/aes/src/backends/aarch64_aes.rs new file mode 100644 index 000000000..8e8ac2542 --- /dev/null +++ b/aes/src/backends/aarch64_aes.rs @@ -0,0 +1,223 @@ +use cipher::{ + Block, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherEncBackend, + BlockCipherEncClosure, BlockSizeUser, ParBlocks, ParBlocksSizeUser, + consts::{U8, U16}, + inout::InOut, +}; + +mod encdec; +mod expand; + +#[cfg(feature = "hazmat")] +pub(crate) mod hazmat; + +use expand::RoundKeys; + +type ParBlocksSize = U8; + +pub(crate) type Aes128 = Aes<11>; +pub(crate) type Aes192 = Aes<13>; +pub(crate) type Aes256 = Aes<15>; + +pub(crate) type Aes128Enc = AesEnc<11>; +pub(crate) type Aes192Enc = AesEnc<13>; +pub(crate) type Aes256Enc = AesEnc<15>; + +pub(crate) type Aes128Dec = AesDec<11>; +pub(crate) type Aes192Dec = AesDec<13>; +pub(crate) type Aes256Dec = AesDec<15>; + +#[derive(Clone, Copy)] +pub(crate) struct Aes { + enc_rk: RoundKeys, + dec_rk: RoundKeys, +} + +impl Aes { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + f.call(self); + } + + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + f.call(self); + } +} + +impl BlockSizeUser for Aes { + type BlockSize = U16; +} + +impl ParBlocksSizeUser for Aes { + type ParBlocksSize = ParBlocksSize; +} + +impl BlockCipherEncBackend for Aes { + #[inline(always)] + fn encrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt(&self.enc_rk, block) }; + } + + #[inline(always)] + fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + } +} + +impl BlockCipherDecBackend for Aes { + #[inline(always)] + fn decrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt(&self.dec_rk, block) }; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt_par(&self.dec_rk, blocks) }; + } +} + +#[derive(Clone, Copy)] +pub(crate) struct AesEnc { + enc_rk: RoundKeys, +} + +impl AesEnc { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn as_encdec(&self) -> Aes { + let enc_rk = self.enc_rk; + let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + Aes { enc_rk, dec_rk } + } + + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn as_dec(&self) -> AesDec { + let dec_rk = unsafe { expand::inv_expanded_keys(&self.enc_rk) }; + AesDec { dec_rk } + } + + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + f.call(self) + } +} + +impl BlockSizeUser for AesEnc { + type BlockSize = U16; +} + +impl ParBlocksSizeUser for AesEnc { + type ParBlocksSize = ParBlocksSize; +} + +impl BlockCipherEncBackend for AesEnc { + #[inline(always)] + fn encrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt(&self.enc_rk, block) }; + } + + #[inline(always)] + fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + } +} + +#[derive(Clone, Copy)] +pub(crate) struct AesDec { + dec_rk: RoundKeys, +} + +impl AesDec { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + f.call(self); + } +} + +impl BlockSizeUser for AesDec { + type BlockSize = U16; +} + +impl ParBlocksSizeUser for AesDec { + type ParBlocksSize = ParBlocksSize; +} + +impl BlockCipherDecBackend for AesDec { + #[inline(always)] + fn decrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt(&self.dec_rk, block) }; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt_par(&self.dec_rk, blocks) }; + } +} + +macro_rules! impl_key_init { + ($name:ty, $name_enc:ty, $name_dec:ty, $key_size:literal) => { + impl $name { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = unsafe { expand::expand_key(key) }; + let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + Self { enc_rk, dec_rk } + } + } + + impl $name_enc { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = unsafe { expand::expand_key(key) }; + Self { enc_rk } + } + } + + impl $name_dec { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = unsafe { expand::expand_key(key) }; + let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + Self { dec_rk } + } + } + }; +} + +impl_key_init!(Aes128, Aes128Enc, Aes128Dec, 16); +impl_key_init!(Aes192, Aes192Enc, Aes192Dec, 24); +impl_key_init!(Aes256, Aes256Enc, Aes256Dec, 32); diff --git a/aes/src/armv8/encdec.rs b/aes/src/backends/aarch64_aes/encdec.rs similarity index 99% rename from aes/src/armv8/encdec.rs rename to aes/src/backends/aarch64_aes/encdec.rs index 913e95cf9..d8175c9bb 100644 --- a/aes/src/armv8/encdec.rs +++ b/aes/src/backends/aarch64_aes/encdec.rs @@ -13,6 +13,7 @@ use core::{arch::aarch64::*, mem}; /// Perform AES encryption using the given expanded keys. #[target_feature(enable = "aes")] +#[inline] pub(super) unsafe fn encrypt( keys: &[uint8x16_t; KEYS], block: InOut<'_, '_, Block>, @@ -38,6 +39,7 @@ pub(super) unsafe fn encrypt( /// Perform AES decryption using the given expanded keys. #[target_feature(enable = "aes")] +#[inline] pub(super) unsafe fn decrypt( keys: &[uint8x16_t; KEYS], block: InOut<'_, '_, Block>, @@ -64,6 +66,7 @@ pub(super) unsafe fn decrypt( /// Perform parallel AES encryption 8-blocks-at-a-time using the given expanded keys. #[target_feature(enable = "aes")] +#[inline] pub(super) unsafe fn encrypt_par( keys: &[uint8x16_t; KEYS], blocks: InOut<'_, '_, Array>, @@ -122,6 +125,7 @@ pub(super) unsafe fn encrypt_par( /// Perform parallel AES decryption 8-blocks-at-a-time using the given expanded keys. #[target_feature(enable = "aes")] +#[inline] pub(super) unsafe fn decrypt_par( keys: &[uint8x16_t; KEYS], blocks: InOut<'_, '_, Array>, diff --git a/aes/src/armv8/expand.rs b/aes/src/backends/aarch64_aes/expand.rs similarity index 69% rename from aes/src/armv8/expand.rs rename to aes/src/backends/aarch64_aes/expand.rs index 1136b2a91..1623c9ebe 100644 --- a/aes/src/armv8/expand.rs +++ b/aes/src/backends/aarch64_aes/expand.rs @@ -3,9 +3,7 @@ use core::{arch::aarch64::*, mem, slice}; -pub(super) type Aes128RoundKeys = [uint8x16_t; 11]; -pub(super) type Aes192RoundKeys = [uint8x16_t; 13]; -pub(super) type Aes256RoundKeys = [uint8x16_t; 15]; +pub(super) type RoundKeys = [uint8x16_t; RK]; /// There are 4 AES words in a block. const BLOCK_WORDS: usize = 4; @@ -17,16 +15,17 @@ const WORD_SIZE: usize = 4; const ROUND_CONSTS: [u32; 10] = [0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36]; /// AES key expansion. +#[inline] #[target_feature(enable = "aes")] -pub unsafe fn expand_key(key: &[u8; L]) -> [uint8x16_t; N] { - assert!((L == 16 && N == 11) || (L == 24 && N == 13) || (L == 32 && N == 15)); +pub unsafe fn expand_key(key: &[u8; N]) -> [uint8x16_t; RK] { + const { assert!(matches!((N, RK), (16, 11) | (24, 13) | (32, 15))) } - let mut expanded_keys: [uint8x16_t; N] = mem::zeroed(); + let mut expanded_keys: [uint8x16_t; RK] = mem::zeroed(); // Sanity check, as this is required in order for the subsequent conversion to be sound. const _: () = assert!(mem::align_of::() >= mem::align_of::()); let keys_ptr: *mut u32 = expanded_keys.as_mut_ptr().cast(); - let columns = slice::from_raw_parts_mut(keys_ptr, N * BLOCK_WORDS); + let columns = slice::from_raw_parts_mut(keys_ptr, RK * BLOCK_WORDS); for (i, chunk) in key.chunks_exact(WORD_SIZE).enumerate() { columns[i] = u32::from_ne_bytes(chunk.try_into().unwrap()); @@ -35,9 +34,9 @@ pub unsafe fn expand_key(key: &[u8; L]) -> [uint // From "The Rijndael Block Cipher" Section 4.1: // > The number of columns of the Cipher Key is denoted by `Nk` and is // > equal to the key length divided by 32 [bits]. - let nk = L / WORD_SIZE; + let nk = N / WORD_SIZE; - for i in nk..(N * BLOCK_WORDS) { + for i in nk..(RK * BLOCK_WORDS) { let mut word = columns[i - 1]; if i % nk == 0 { @@ -56,16 +55,19 @@ pub unsafe fn expand_key(key: &[u8; L]) -> [uint /// /// This is the reverse of the encryption keys, with the Inverse Mix Columns /// operation applied to all but the first and last expanded key. +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn inv_expanded_keys(keys: &[uint8x16_t; N]) -> [uint8x16_t; N] { - assert!(N == 11 || N == 13 || N == 15); - - let mut inv_keys: [uint8x16_t; N] = core::mem::zeroed(); - inv_keys[0] = keys[N - 1]; - for i in 1..N - 1 { - inv_keys[i] = vaesimcq_u8(keys[N - 1 - i]); +pub(super) unsafe fn inv_expanded_keys( + keys: &[uint8x16_t; RK], +) -> [uint8x16_t; RK] { + const { assert!(matches!(RK, 11 | 13 | 15)) } + + let mut inv_keys: [uint8x16_t; RK] = core::mem::zeroed(); + inv_keys[0] = keys[RK - 1]; + for i in 1..RK - 1 { + inv_keys[i] = vaesimcq_u8(keys[RK - 1 - i]); } - inv_keys[N - 1] = keys[0]; + inv_keys[RK - 1] = keys[0]; inv_keys } diff --git a/aes/src/armv8/hazmat.rs b/aes/src/backends/aarch64_aes/hazmat.rs similarity index 100% rename from aes/src/armv8/hazmat.rs rename to aes/src/backends/aarch64_aes/hazmat.rs diff --git a/aes/src/backends/fixslice/aes128.rs b/aes/src/backends/fixslice/aes128.rs new file mode 100644 index 000000000..21efc8bb3 --- /dev/null +++ b/aes/src/backends/fixslice/aes128.rs @@ -0,0 +1,167 @@ +use super::{BatchBlocks, State, Word, mix_columns::*, sbox::*, utils::*}; + +/// AES-128 round keys +pub(crate) type RoundKeys = [W; 88]; + +/// Fully bitsliced AES-128 key schedule to match the fully-fixsliced representation. +pub(crate) fn key_schedule(key: &[u8; 16]) -> RoundKeys { + let mut rkeys = [W::default(); 88]; + + W::bitslice(&mut rkeys[..8], &broadcast::(key)); + + let mut rk_off = 0; + for rcon in 0..10 { + memshift32(&mut rkeys, rk_off); + rk_off += 8; + + sub_bytes(&mut rkeys[rk_off..(rk_off + 8)]); + sub_bytes_nots(&mut rkeys[rk_off..(rk_off + 8)]); + + if rcon < 8 { + add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon); + } else { + add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 8); + add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 7); + add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 5); + add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 4); + } + + xor_columns(&mut rkeys, rk_off, 8, W::ror_distance(1, 3)); + } + + // Adjust to match fixslicing format + #[cfg(aes_backend_soft = "compact")] + { + for i in (8..88).step_by(16) { + inv_shift_rows_1(&mut rkeys[i..(i + 8)]); + } + } + #[cfg(not(aes_backend_soft = "compact"))] + { + for i in (8..72).step_by(32) { + inv_shift_rows_1(&mut rkeys[i..(i + 8)]); + inv_shift_rows_2(&mut rkeys[(i + 8)..(i + 16)]); + inv_shift_rows_3(&mut rkeys[(i + 16)..(i + 24)]); + } + inv_shift_rows_1(&mut rkeys[72..80]); + } + + // Account for NOTs removed from sub_bytes + for i in 1..11 { + sub_bytes_nots(&mut rkeys[(i * 8)..(i * 8 + 8)]); + } + + rkeys +} + +/// Fully-fixsliced AES-128 encryption (the ShiftRows is completely omitted). +/// +/// Encrypts four blocks in-place and in parallel. +pub(crate) fn encrypt(rkeys: &RoundKeys, blocks: &BatchBlocks) -> BatchBlocks { + let mut state = State::::default(); + + W::bitslice(&mut state, blocks); + + add_round_key(&mut state, &rkeys[..8]); + + let mut rk_off = 8; + loop { + sub_bytes(&mut state); + mix_columns_1(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + + #[cfg(aes_backend_soft = "compact")] + { + shift_rows_2(&mut state); + } + + if rk_off == 80 { + break; + } + + #[cfg(not(aes_backend_soft = "compact"))] + { + sub_bytes(&mut state); + mix_columns_2(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + + sub_bytes(&mut state); + mix_columns_3(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + } + + sub_bytes(&mut state); + mix_columns_0(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + } + + #[cfg(not(aes_backend_soft = "compact"))] + { + shift_rows_2(&mut state); + } + + sub_bytes(&mut state); + add_round_key(&mut state, &rkeys[80..]); + + W::inv_bitslice(&state) +} + +/// Fully-fixsliced AES-128 decryption (the InvShiftRows is completely omitted). +/// +/// Decrypts four blocks in-place and in parallel. +pub(crate) fn decrypt(rkeys: &RoundKeys, blocks: &BatchBlocks) -> BatchBlocks { + let mut state = State::::default(); + + W::bitslice(&mut state, blocks); + + add_round_key(&mut state, &rkeys[80..]); + inv_sub_bytes(&mut state); + + #[cfg(not(aes_backend_soft = "compact"))] + { + inv_shift_rows_2(&mut state); + } + + let mut rk_off = 72; + loop { + #[cfg(aes_backend_soft = "compact")] + { + inv_shift_rows_2(&mut state); + } + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_1(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + if rk_off == 0 { + break; + } + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_0(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + #[cfg(not(aes_backend_soft = "compact"))] + { + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_3(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_2(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + } + } + + add_round_key(&mut state, &rkeys[..8]); + + W::inv_bitslice(&state) +} diff --git a/aes/src/backends/fixslice/aes192.rs b/aes/src/backends/fixslice/aes192.rs new file mode 100644 index 000000000..0edf37f1e --- /dev/null +++ b/aes/src/backends/fixslice/aes192.rs @@ -0,0 +1,203 @@ +use super::{BatchBlocks, State, Word, mix_columns::*, sbox::*, utils::*}; + +/// AES-192 round keys +pub(crate) type RoundKeys = [W; 104]; + +/// Fully bitsliced AES-192 key schedule to match the fully-fixsliced representation. +pub(crate) fn key_schedule(key: &[u8; 24]) -> RoundKeys { + let mut rkeys = [W::default(); 104]; + let mut tmp = [W::default(); 8]; + + W::bitslice(&mut rkeys[..8], &broadcast::(&key[..16])); + W::bitslice(&mut tmp, &broadcast::(&key[8..])); + + let mut rcon = 0; + let mut rk_off = 8; + + loop { + for i in 0..8 { + rkeys[rk_off + i] = (W::uniform_row(0x0f) & (tmp[i] >> W::HALF_ROW)) + | (W::uniform_row(0xf0) & (rkeys[(rk_off - 8) + i] << W::HALF_ROW)); + } + + sub_bytes(&mut tmp); + sub_bytes_nots(&mut tmp); + + add_round_constant_bit(&mut tmp, rcon); + rcon += 1; + + for i in 0..8 { + let mut ti = rkeys[rk_off + i]; + ti ^= W::uniform_row(0x30) & tmp[i].ror(W::ror_distance(1, 1)); + ti ^= W::uniform_row(0xc0) & (ti << W::QUARTER_ROW); + tmp[i] = ti; + } + rkeys[rk_off..(rk_off + 8)].copy_from_slice(&tmp); + rk_off += 8; + + for i in 0..8 { + let ui = tmp[i]; + let mut ti = (W::uniform_row(0x0f) & (rkeys[(rk_off - 16) + i] >> W::HALF_ROW)) + | (W::uniform_row(0xf0) & (ui << W::HALF_ROW)); + ti ^= W::uniform_row(0x03) & (ui >> (3 * W::QUARTER_ROW)); + tmp[i] = ti + ^ (W::uniform_row(0xfc) & (ti << W::QUARTER_ROW)) + ^ (W::uniform_row(0xf0) & (ti << W::HALF_ROW)) + ^ (W::uniform_row(0xc0) & (ti << (3 * W::QUARTER_ROW))); + } + rkeys[rk_off..(rk_off + 8)].copy_from_slice(&tmp); + rk_off += 8; + + sub_bytes(&mut tmp); + sub_bytes_nots(&mut tmp); + + add_round_constant_bit(&mut tmp, rcon); + rcon += 1; + + for i in 0..8 { + let mut ti = (W::uniform_row(0x0f) & (rkeys[(rk_off - 16) + i] >> W::HALF_ROW)) + | (W::uniform_row(0xf0) & (rkeys[(rk_off - 8) + i] << W::HALF_ROW)); + ti ^= W::uniform_row(0x03) & tmp[i].ror(W::ror_distance(1, 3)); + rkeys[rk_off + i] = ti + ^ (W::uniform_row(0xfc) & (ti << W::QUARTER_ROW)) + ^ (W::uniform_row(0xf0) & (ti << W::HALF_ROW)) + ^ (W::uniform_row(0xc0) & (ti << (3 * W::QUARTER_ROW))); + } + rk_off += 8; + + if rcon >= 8 { + break; + } + + for i in 0..8 { + let ui = rkeys[(rk_off - 8) + i]; + let mut ti = rkeys[(rk_off - 16) + i]; + ti ^= W::uniform_row(0x30) & (ui >> W::QUARTER_ROW); + ti ^= W::uniform_row(0xc0) & (ti << W::QUARTER_ROW); + tmp[i] = ti; + } + } + + // Adjust to match fixslicing format + #[cfg(aes_backend_soft = "compact")] + { + for i in (8..104).step_by(16) { + inv_shift_rows_1(&mut rkeys[i..(i + 8)]); + } + } + #[cfg(not(aes_backend_soft = "compact"))] + { + for i in (0..96).step_by(32) { + inv_shift_rows_1(&mut rkeys[(i + 8)..(i + 16)]); + inv_shift_rows_2(&mut rkeys[(i + 16)..(i + 24)]); + inv_shift_rows_3(&mut rkeys[(i + 24)..(i + 32)]); + } + } + + // Account for NOTs removed from sub_bytes + for i in 1..13 { + sub_bytes_nots(&mut rkeys[(i * 8)..(i * 8 + 8)]); + } + + rkeys +} + +/// Fully-fixsliced AES-192 encryption (the ShiftRows is completely omitted). +/// +/// Encrypts four blocks in-place and in parallel. +pub(crate) fn encrypt(rkeys: &RoundKeys, blocks: &BatchBlocks) -> BatchBlocks { + let mut state = State::::default(); + + W::bitslice(&mut state, blocks); + + add_round_key(&mut state, &rkeys[..8]); + + let mut rk_off = 8; + loop { + sub_bytes(&mut state); + mix_columns_1(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + + #[cfg(aes_backend_soft = "compact")] + { + shift_rows_2(&mut state); + } + #[cfg(not(aes_backend_soft = "compact"))] + { + sub_bytes(&mut state); + mix_columns_2(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + + sub_bytes(&mut state); + mix_columns_3(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + } + + if rk_off == 96 { + break; + } + + sub_bytes(&mut state); + mix_columns_0(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + } + + sub_bytes(&mut state); + add_round_key(&mut state, &rkeys[96..]); + + W::inv_bitslice(&state) +} + +/// Fully-fixsliced AES-192 decryption (the InvShiftRows is completely omitted). +/// +/// Decrypts four blocks in-place and in parallel. +pub(crate) fn decrypt(rkeys: &RoundKeys, blocks: &BatchBlocks) -> BatchBlocks { + let mut state = State::::default(); + + W::bitslice(&mut state, blocks); + + add_round_key(&mut state, &rkeys[96..]); + inv_sub_bytes(&mut state); + + let mut rk_off = 88; + loop { + #[cfg(aes_backend_soft = "compact")] + { + inv_shift_rows_2(&mut state); + } + #[cfg(not(aes_backend_soft = "compact"))] + { + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_3(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_2(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + } + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_1(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + if rk_off == 0 { + break; + } + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_0(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + } + + add_round_key(&mut state, &rkeys[..8]); + + W::inv_bitslice(&state) +} diff --git a/aes/src/backends/fixslice/aes256.rs b/aes/src/backends/fixslice/aes256.rs new file mode 100644 index 000000000..82576d977 --- /dev/null +++ b/aes/src/backends/fixslice/aes256.rs @@ -0,0 +1,175 @@ +use super::{BatchBlocks, State, Word, mix_columns::*, sbox::*, utils::*}; + +/// AES-256 round keys +pub(crate) type RoundKeys = [W; 120]; + +/// Fully bitsliced AES-256 key schedule to match the fully-fixsliced representation. +pub(crate) fn key_schedule(key: &[u8; 32]) -> RoundKeys { + let mut rkeys = [W::default(); 120]; + + W::bitslice(&mut rkeys[..8], &broadcast::(&key[..16])); + W::bitslice(&mut rkeys[8..16], &broadcast::(&key[16..])); + + let mut rk_off = 8; + + let mut rcon = 0; + loop { + memshift32(&mut rkeys, rk_off); + rk_off += 8; + + sub_bytes(&mut rkeys[rk_off..(rk_off + 8)]); + sub_bytes_nots(&mut rkeys[rk_off..(rk_off + 8)]); + + add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon); + xor_columns(&mut rkeys, rk_off, 16, W::ror_distance(1, 3)); + rcon += 1; + + if rcon == 7 { + break; + } + + memshift32(&mut rkeys, rk_off); + rk_off += 8; + + sub_bytes(&mut rkeys[rk_off..(rk_off + 8)]); + sub_bytes_nots(&mut rkeys[rk_off..(rk_off + 8)]); + + xor_columns(&mut rkeys, rk_off, 16, W::ror_distance(0, 3)); + } + + // Adjust to match fixslicing format + #[cfg(aes_backend_soft = "compact")] + { + for i in (8..120).step_by(16) { + inv_shift_rows_1(&mut rkeys[i..(i + 8)]); + } + } + #[cfg(not(aes_backend_soft = "compact"))] + { + for i in (8..104).step_by(32) { + inv_shift_rows_1(&mut rkeys[i..(i + 8)]); + inv_shift_rows_2(&mut rkeys[(i + 8)..(i + 16)]); + inv_shift_rows_3(&mut rkeys[(i + 16)..(i + 24)]); + } + inv_shift_rows_1(&mut rkeys[104..112]); + } + + // Account for NOTs removed from sub_bytes + for i in 1..15 { + sub_bytes_nots(&mut rkeys[(i * 8)..(i * 8 + 8)]); + } + + rkeys +} + +/// Fully-fixsliced AES-256 encryption (the ShiftRows is completely omitted). +/// +/// Encrypts four blocks in-place and in parallel. +pub(crate) fn encrypt(rkeys: &RoundKeys, blocks: &BatchBlocks) -> BatchBlocks { + let mut state = State::::default(); + + W::bitslice(&mut state, blocks); + + add_round_key(&mut state, &rkeys[..8]); + + let mut rk_off = 8; + loop { + sub_bytes(&mut state); + mix_columns_1(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + + #[cfg(aes_backend_soft = "compact")] + { + shift_rows_2(&mut state); + } + + if rk_off == 112 { + break; + } + + #[cfg(not(aes_backend_soft = "compact"))] + { + sub_bytes(&mut state); + mix_columns_2(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + + sub_bytes(&mut state); + mix_columns_3(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + } + + sub_bytes(&mut state); + mix_columns_0(&mut state); + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + rk_off += 8; + } + + #[cfg(not(aes_backend_soft = "compact"))] + { + shift_rows_2(&mut state); + } + + sub_bytes(&mut state); + add_round_key(&mut state, &rkeys[112..]); + + W::inv_bitslice(&state) +} + +/// Fully-fixsliced AES-256 decryption (the InvShiftRows is completely omitted). +/// +/// Decrypts four blocks in-place and in parallel. +pub(crate) fn decrypt(rkeys: &RoundKeys, blocks: &BatchBlocks) -> BatchBlocks { + let mut state = State::::default(); + + W::bitslice(&mut state, blocks); + + add_round_key(&mut state, &rkeys[112..]); + inv_sub_bytes(&mut state); + + #[cfg(not(aes_backend_soft = "compact"))] + { + inv_shift_rows_2(&mut state); + } + + let mut rk_off = 104; + loop { + #[cfg(aes_backend_soft = "compact")] + { + inv_shift_rows_2(&mut state); + } + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_1(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + if rk_off == 0 { + break; + } + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_0(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + #[cfg(not(aes_backend_soft = "compact"))] + { + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_3(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + + add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); + inv_mix_columns_2(&mut state); + inv_sub_bytes(&mut state); + rk_off -= 8; + } + } + + add_round_key(&mut state, &rkeys[..8]); + + W::inv_bitslice(&state) +} diff --git a/aes/src/backends/fixslice/hazmat.rs b/aes/src/backends/fixslice/hazmat.rs new file mode 100644 index 000000000..5be60a20d --- /dev/null +++ b/aes/src/backends/fixslice/hazmat.rs @@ -0,0 +1,138 @@ +use super::{BatchBlocks, NativeWord, State, Word, mix_columns::*, sbox::*, utils::*}; +use crate::hazmat::{Block, Block8}; +use cipher::typenum::Unsigned; + +/// XOR the `src` block into the `dst` block in-place. +fn xor_in_place(dst: &mut Block, src: &Block) { + for (a, b) in dst.iter_mut().zip(src.as_slice()) { + *a ^= *b; + } +} + +fn cipher_round_generic(block: &mut Block, round_key: &Block) { + let mut state = State::::default(); + W::bitslice(&mut state, &broadcast::(block.as_slice())); + sub_bytes(&mut state); + sub_bytes_nots(&mut state); + shift_rows_1(&mut state); + mix_columns_0(&mut state); + inv_bitslice_one(block, &state); + xor_in_place(block, round_key); +} + +fn cipher_round_par_generic(blocks: &mut Block8, round_keys: &Block8) { + let blocks_per_batch = <::Blocks>::USIZE; + for (chunk, keys) in blocks + .chunks_exact_mut(blocks_per_batch) + .zip(round_keys.chunks_exact(blocks_per_batch)) + { + let mut state = State::::default(); + let mut batch = BatchBlocks::::default(); + for (slot, blk) in batch.iter_mut().zip(chunk.iter()) { + slot.copy_from_slice(blk.as_slice()); + } + W::bitslice(&mut state, &batch); + sub_bytes(&mut state); + sub_bytes_nots(&mut state); + shift_rows_1(&mut state); + mix_columns_0(&mut state); + let res = W::inv_bitslice(&state); + + for i in 0..blocks_per_batch { + chunk[i] = res[i]; + xor_in_place(&mut chunk[i], &keys[i]); + } + } +} + +fn equiv_inv_cipher_round_generic(block: &mut Block, round_key: &Block) { + let mut state = State::::default(); + W::bitslice(&mut state, &broadcast::(block.as_slice())); + sub_bytes_nots(&mut state); + inv_sub_bytes(&mut state); + inv_shift_rows_1(&mut state); + inv_mix_columns_0(&mut state); + inv_bitslice_one(block, &state); + xor_in_place(block, round_key); +} + +fn equiv_inv_cipher_round_par_generic(blocks: &mut Block8, round_keys: &Block8) { + let blocks_per_batch = <::Blocks>::USIZE; + for (chunk, keys) in blocks + .chunks_exact_mut(blocks_per_batch) + .zip(round_keys.chunks_exact(blocks_per_batch)) + { + let mut state = State::::default(); + let mut batch = BatchBlocks::::default(); + for (slot, blk) in batch.iter_mut().zip(chunk.iter()) { + slot.copy_from_slice(blk.as_slice()); + } + W::bitslice(&mut state, &batch); + sub_bytes_nots(&mut state); + inv_sub_bytes(&mut state); + inv_shift_rows_1(&mut state); + inv_mix_columns_0(&mut state); + let res = W::inv_bitslice(&state); + + for i in 0..blocks_per_batch { + chunk[i] = res[i]; + xor_in_place(&mut chunk[i], &keys[i]); + } + } +} + +fn mix_columns_generic(block: &mut Block) { + let mut state = State::::default(); + W::bitslice(&mut state, &broadcast::(block.as_slice())); + mix_columns_0(&mut state); + inv_bitslice_one(block, &state); +} + +fn inv_mix_columns_generic(block: &mut Block) { + let mut state = State::::default(); + W::bitslice(&mut state, &broadcast::(block.as_slice())); + inv_mix_columns_0(&mut state); + inv_bitslice_one(block, &state); +} + +/// AES cipher (encrypt) round function. +#[inline] +pub(crate) fn cipher_round(block: &mut Block, round_key: &Block) { + cipher_round_generic::(block, round_key) +} + +/// AES cipher (encrypt) round function: parallel version. +#[inline] +pub(crate) fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { + cipher_round_par_generic::(blocks, round_keys) +} + +/// AES cipher (encrypt) inverse round function. +#[inline] +pub(crate) fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { + equiv_inv_cipher_round_generic::(block, round_key) +} + +/// AES cipher (encrypt) inverse round function: parallel version. +#[inline] +pub(crate) fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { + equiv_inv_cipher_round_par_generic::(blocks, round_keys) +} + +/// AES mix columns function. +#[inline] +pub(crate) fn mix_columns(block: &mut Block) { + mix_columns_generic::(block) +} + +/// AES inverse mix columns function. +#[inline] +pub(crate) fn inv_mix_columns(block: &mut Block) { + inv_mix_columns_generic::(block) +} + +#[inline] +fn inv_bitslice_one(block: &mut Block, state: &State) { + let out = W::inv_bitslice(state); + block.copy_from_slice(out[0].as_slice()); +} diff --git a/aes/src/backends/fixslice/mix_columns.rs b/aes/src/backends/fixslice/mix_columns.rs new file mode 100644 index 000000000..c5a7bc7ba --- /dev/null +++ b/aes/src/backends/fixslice/mix_columns.rs @@ -0,0 +1,174 @@ +use super::{State, Word}; + +/// Computation of the MixColumns transformation in the fixsliced representation, +/// with different rotations used according to the round number mod 4. +/// +/// Based on Käsper-Schwabe, similar to https://github.com/Ko-/aes-armcortexm. +macro_rules! define_mix_columns { + ( + $name:ident, + $name_inv:ident, + $first_rotate:path, + $second_rotate:path + ) => { + #[rustfmt::skip] + pub(crate) fn $name(state: &mut State) { + let (a0, a1, a2, a3, a4, a5, a6, a7) = ( + state[0], state[1], state[2], state[3], state[4], state[5], state[6], state[7] + ); + let (b0, b1, b2, b3, b4, b5, b6, b7) = ( + $first_rotate(a0), + $first_rotate(a1), + $first_rotate(a2), + $first_rotate(a3), + $first_rotate(a4), + $first_rotate(a5), + $first_rotate(a6), + $first_rotate(a7), + ); + let (c0, c1, c2, c3, c4, c5, c6, c7) = ( + a0 ^ b0, + a1 ^ b1, + a2 ^ b2, + a3 ^ b3, + a4 ^ b4, + a5 ^ b5, + a6 ^ b6, + a7 ^ b7, + ); + state[0] = b0 ^ c7 ^ $second_rotate(c0); + state[1] = b1 ^ c0 ^ c7 ^ $second_rotate(c1); + state[2] = b2 ^ c1 ^ $second_rotate(c2); + state[3] = b3 ^ c2 ^ c7 ^ $second_rotate(c3); + state[4] = b4 ^ c3 ^ c7 ^ $second_rotate(c4); + state[5] = b5 ^ c4 ^ $second_rotate(c5); + state[6] = b6 ^ c5 ^ $second_rotate(c6); + state[7] = b7 ^ c6 ^ $second_rotate(c7); + } + + #[rustfmt::skip] + pub(crate) fn $name_inv(state: &mut State) { + let (a0, a1, a2, a3, a4, a5, a6, a7) = ( + state[0], state[1], state[2], state[3], state[4], state[5], state[6], state[7] + ); + let (b0, b1, b2, b3, b4, b5, b6, b7) = ( + $first_rotate(a0), + $first_rotate(a1), + $first_rotate(a2), + $first_rotate(a3), + $first_rotate(a4), + $first_rotate(a5), + $first_rotate(a6), + $first_rotate(a7), + ); + let (c0, c1, c2, c3, c4, c5, c6, c7) = ( + a0 ^ b0, + a1 ^ b1, + a2 ^ b2, + a3 ^ b3, + a4 ^ b4, + a5 ^ b5, + a6 ^ b6, + a7 ^ b7, + ); + let (d0, d1, d2, d3, d4, d5, d6, d7) = ( + a0 ^ c7, + a1 ^ c0 ^ c7, + a2 ^ c1, + a3 ^ c2 ^ c7, + a4 ^ c3 ^ c7, + a5 ^ c4, + a6 ^ c5, + a7 ^ c6, + ); + let (e0, e1, e2, e3, e4, e5, e6, e7) = ( + c0 ^ d6, + c1 ^ d6 ^ d7, + c2 ^ d0 ^ d7, + c3 ^ d1 ^ d6, + c4 ^ d2 ^ d6 ^ d7, + c5 ^ d3 ^ d7, + c6 ^ d4, + c7 ^ d5, + ); + state[0] = d0 ^ e0 ^ $second_rotate(e0); + state[1] = d1 ^ e1 ^ $second_rotate(e1); + state[2] = d2 ^ e2 ^ $second_rotate(e2); + state[3] = d3 ^ e3 ^ $second_rotate(e3); + state[4] = d4 ^ e4 ^ $second_rotate(e4); + state[5] = d5 ^ e5 ^ $second_rotate(e5); + state[6] = d6 ^ e6 ^ $second_rotate(e6); + state[7] = d7 ^ e7 ^ $second_rotate(e7); + } + }; +} + +define_mix_columns!( + mix_columns_0, + inv_mix_columns_0, + rotate_rows_1, + rotate_rows_2 +); + +define_mix_columns!( + mix_columns_1, + inv_mix_columns_1, + rotate_rows_and_columns_1_1, + rotate_rows_and_columns_2_2 +); + +#[cfg(not(aes_backend_soft = "compact"))] +define_mix_columns!( + mix_columns_2, + inv_mix_columns_2, + rotate_rows_and_columns_1_2, + rotate_rows_2 +); + +#[cfg(not(aes_backend_soft = "compact"))] +define_mix_columns!( + mix_columns_3, + inv_mix_columns_3, + rotate_rows_and_columns_1_3, + rotate_rows_and_columns_2_2 +); + +#[inline(always)] +fn rotate_rows_1(x: W) -> W { + x.ror(W::ror_distance(1, 0)) +} + +#[inline(always)] +fn rotate_rows_2(x: W) -> W { + x.ror(W::ror_distance(2, 0)) +} + +#[inline(always)] +fn rotate_rows_and_columns_1_1(x: W) -> W { + let a = x.ror(W::ror_distance(1, 1)) & W::uniform_row(0x3f); + let b = x.ror(W::ror_distance(0, 1)) & W::uniform_row(0xc0); + a | b +} + +#[cfg(not(aes_backend_soft = "compact"))] +#[inline(always)] +fn rotate_rows_and_columns_1_2(x: W) -> W { + let a = x.ror(W::ror_distance(1, 2)) & W::uniform_row(0x0f); + let b = x.ror(W::ror_distance(0, 2)) & W::uniform_row(0xf0); + a | b +} + +#[cfg(not(aes_backend_soft = "compact"))] +#[inline(always)] +fn rotate_rows_and_columns_1_3(x: W) -> W { + let a = x.ror(W::ror_distance(1, 3)) & W::uniform_row(0x03); + let b = x.ror(W::ror_distance(0, 3)) & W::uniform_row(0xfc); + a | b +} + +#[inline(always)] +fn rotate_rows_and_columns_2_2(x: W) -> W { + let a = x.ror(W::ror_distance(2, 2)) & W::uniform_row(0x0f); + let b = x.ror(W::ror_distance(1, 2)) & W::uniform_row(0xf0); + a | b +} diff --git a/aes/src/backends/fixslice/mod.rs b/aes/src/backends/fixslice/mod.rs new file mode 100644 index 000000000..e4146286d --- /dev/null +++ b/aes/src/backends/fixslice/mod.rs @@ -0,0 +1,43 @@ +//! Fixsliced implementations of AES-128, AES-192 and AES-256 (64-bit) +//! adapted from the C implementation. +//! +//! All implementations are fully bitsliced and do not rely on any +//! Look-Up Table (LUT). +//! +//! See the paper at for more details. +//! +//! # Author (original C code) +//! +//! Alexandre Adomnicai, Nanyang Technological University, Singapore +//! +//! +//! Originally licensed MIT. Relicensed as Apache 2.0+MIT with permission. + +use cipher::array::Array; + +pub(crate) mod aes128; +pub(crate) mod aes192; +pub(crate) mod aes256; +#[cfg(feature = "hazmat")] +pub(crate) mod hazmat; + +mod mix_columns; +mod sbox; +mod utils; +mod word; + +use word::Word; + +type State = [W; 8]; + +cpubits::cpubits! { + 16 | 32 => { + pub(crate) type NativeWord = u32; + } + 64 => { + pub(crate) type NativeWord = u64; + } +} + +pub(crate) type NativeBatchSize = ::Blocks; +pub(crate) type BatchBlocks = Array::Blocks>; diff --git a/aes/src/backends/fixslice/sbox.rs b/aes/src/backends/fixslice/sbox.rs new file mode 100644 index 000000000..b65fcc8c8 --- /dev/null +++ b/aes/src/backends/fixslice/sbox.rs @@ -0,0 +1,395 @@ +//! S-box and inverse S-box (Boyar-Peralta-Calik) + +use super::Word; + +/// Bitsliced implementation of the AES Sbox based on Boyar, Peralta and Calik. +/// +/// See: +/// +/// Note that the 4 bitwise NOT are moved to the key schedule. +pub(crate) fn sub_bytes(state: &mut [W]) { + debug_assert_eq!(state.len(), 8); + + // Scheduled using https://github.com/Ko-/aes-armcortexm/tree/public/scheduler + // Inline "stack" comments reflect suggested stores and loads (ARM Cortex-M3 and M4) + + let u7 = state[0]; + let u6 = state[1]; + let u5 = state[2]; + let u4 = state[3]; + let u3 = state[4]; + let u2 = state[5]; + let u1 = state[6]; + let u0 = state[7]; + + let y14 = u3 ^ u5; + let y13 = u0 ^ u6; + let y12 = y13 ^ y14; + let t1 = u4 ^ y12; + let y15 = t1 ^ u5; + let t2 = y12 & y15; + let y6 = y15 ^ u7; + let y20 = t1 ^ u1; + // y12 -> stack + let y9 = u0 ^ u3; + // y20 -> stack + let y11 = y20 ^ y9; + // y9 -> stack + let t12 = y9 & y11; + // y6 -> stack + let y7 = u7 ^ y11; + let y8 = u0 ^ u5; + let t0 = u1 ^ u2; + let y10 = y15 ^ t0; + // y15 -> stack + let y17 = y10 ^ y11; + // y14 -> stack + let t13 = y14 & y17; + let t14 = t13 ^ t12; + // y17 -> stack + let y19 = y10 ^ y8; + // y10 -> stack + let t15 = y8 & y10; + let t16 = t15 ^ t12; + let y16 = t0 ^ y11; + // y11 -> stack + let y21 = y13 ^ y16; + // y13 -> stack + let t7 = y13 & y16; + // y16 -> stack + let y18 = u0 ^ y16; + let y1 = t0 ^ u7; + let y4 = y1 ^ u3; + // u7 -> stack + let t5 = y4 & u7; + let t6 = t5 ^ t2; + let t18 = t6 ^ t16; + let t22 = t18 ^ y19; + let y2 = y1 ^ u0; + let t10 = y2 & y7; + let t11 = t10 ^ t7; + let t20 = t11 ^ t16; + let t24 = t20 ^ y18; + let y5 = y1 ^ u6; + let t8 = y5 & y1; + let t9 = t8 ^ t7; + let t19 = t9 ^ t14; + let t23 = t19 ^ y21; + let y3 = y5 ^ y8; + // y6 <- stack + let t3 = y3 & y6; + let t4 = t3 ^ t2; + // y20 <- stack + let t17 = t4 ^ y20; + let t21 = t17 ^ t14; + let t26 = t21 & t23; + let t27 = t24 ^ t26; + let t31 = t22 ^ t26; + let t25 = t21 ^ t22; + // y4 -> stack + let t28 = t25 & t27; + let t29 = t28 ^ t22; + let z14 = t29 & y2; + let z5 = t29 & y7; + let t30 = t23 ^ t24; + let t32 = t31 & t30; + let t33 = t32 ^ t24; + let t35 = t27 ^ t33; + let t36 = t24 & t35; + let t38 = t27 ^ t36; + let t39 = t29 & t38; + let t40 = t25 ^ t39; + let t43 = t29 ^ t40; + // y16 <- stack + let z3 = t43 & y16; + let tc12 = z3 ^ z5; + // tc12 -> stack + // y13 <- stack + let z12 = t43 & y13; + let z13 = t40 & y5; + let z4 = t40 & y1; + let tc6 = z3 ^ z4; + let t34 = t23 ^ t33; + let t37 = t36 ^ t34; + let t41 = t40 ^ t37; + // y10 <- stack + let z8 = t41 & y10; + let z17 = t41 & y8; + let t44 = t33 ^ t37; + // y15 <- stack + let z0 = t44 & y15; + // z17 -> stack + // y12 <- stack + let z9 = t44 & y12; + let z10 = t37 & y3; + let z1 = t37 & y6; + let tc5 = z1 ^ z0; + let tc11 = tc6 ^ tc5; + // y4 <- stack + let z11 = t33 & y4; + let t42 = t29 ^ t33; + let t45 = t42 ^ t41; + // y17 <- stack + let z7 = t45 & y17; + let tc8 = z7 ^ tc6; + // y14 <- stack + let z16 = t45 & y14; + // y11 <- stack + let z6 = t42 & y11; + let tc16 = z6 ^ tc8; + // z14 -> stack + // y9 <- stack + let z15 = t42 & y9; + let tc20 = z15 ^ tc16; + let tc1 = z15 ^ z16; + let tc2 = z10 ^ tc1; + let tc21 = tc2 ^ z11; + let tc3 = z9 ^ tc2; + let s0 = tc3 ^ tc16; + let s3 = tc3 ^ tc11; + let s1 = s3 ^ tc16; + let tc13 = z13 ^ tc1; + // u7 <- stack + let z2 = t33 & u7; + let tc4 = z0 ^ z2; + let tc7 = z12 ^ tc4; + let tc9 = z8 ^ tc7; + let tc10 = tc8 ^ tc9; + // z14 <- stack + let tc17 = z14 ^ tc10; + let s5 = tc21 ^ tc17; + let tc26 = tc17 ^ tc20; + // z17 <- stack + let s2 = tc26 ^ z17; + // tc12 <- stack + let tc14 = tc4 ^ tc12; + let tc18 = tc13 ^ tc14; + let s6 = tc10 ^ tc18; + let s7 = z12 ^ tc18; + let s4 = tc14 ^ s3; + + state[0] = s7; + state[1] = s6; + state[2] = s5; + state[3] = s4; + state[4] = s3; + state[5] = s2; + state[6] = s1; + state[7] = s0; +} + +/// Note that the 4 bitwise NOT are accounted for here so that it is a true +/// inverse of `sub_bytes`. +pub(crate) fn inv_sub_bytes(state: &mut [W]) { + debug_assert_eq!(state.len(), 8); + + // Scheduled using https://github.com/Ko-/aes-armcortexm/tree/public/scheduler + // Inline "stack" comments reflect suggested stores and loads (ARM Cortex-M3 and M4) + + let u7 = state[0]; + let u6 = state[1]; + let u5 = state[2]; + let u4 = state[3]; + let u3 = state[4]; + let u2 = state[5]; + let u1 = state[6]; + let u0 = state[7]; + + let t23 = u0 ^ u3; + let t8 = u1 ^ t23; + let m2 = t23 & t8; + let t4 = u4 ^ t8; + let t22 = u1 ^ u3; + let t2 = u0 ^ u1; + let t1 = u3 ^ u4; + // t23 -> stack + let t9 = u7 ^ t1; + // t8 -> stack + let m7 = t22 & t9; + // t9 -> stack + let t24 = u4 ^ u7; + // m7 -> stack + let t10 = t2 ^ t24; + // u4 -> stack + let m14 = t2 & t10; + let r5 = u6 ^ u7; + // m2 -> stack + let t3 = t1 ^ r5; + // t2 -> stack + let t13 = t2 ^ r5; + let t19 = t22 ^ r5; + // t3 -> stack + let t17 = u2 ^ t19; + // t4 -> stack + let t25 = u2 ^ t1; + let r13 = u1 ^ u6; + // t25 -> stack + let t20 = t24 ^ r13; + // t17 -> stack + let m9 = t20 & t17; + // t20 -> stack + let r17 = u2 ^ u5; + // t22 -> stack + let t6 = t22 ^ r17; + // t13 -> stack + let m1 = t13 & t6; + let y5 = u0 ^ r17; + let m4 = t19 & y5; + let m5 = m4 ^ m1; + let m17 = m5 ^ t24; + let r18 = u5 ^ u6; + let t27 = t1 ^ r18; + let t15 = t10 ^ t27; + // t6 -> stack + let m11 = t1 & t15; + let m15 = m14 ^ m11; + let m21 = m17 ^ m15; + // t1 -> stack + // t4 <- stack + let m12 = t4 & t27; + let m13 = m12 ^ m11; + let t14 = t10 ^ r18; + let m3 = t14 ^ m1; + // m2 <- stack + let m16 = m3 ^ m2; + let m20 = m16 ^ m13; + // u4 <- stack + let r19 = u2 ^ u4; + let t16 = r13 ^ r19; + // t3 <- stack + let t26 = t3 ^ t16; + let m6 = t3 & t16; + let m8 = t26 ^ m6; + // t10 -> stack + // m7 <- stack + let m18 = m8 ^ m7; + let m22 = m18 ^ m13; + let m25 = m22 & m20; + let m26 = m21 ^ m25; + let m10 = m9 ^ m6; + let m19 = m10 ^ m15; + // t25 <- stack + let m23 = m19 ^ t25; + let m28 = m23 ^ m25; + let m24 = m22 ^ m23; + let m30 = m26 & m24; + let m39 = m23 ^ m30; + let m48 = m39 & y5; + let m57 = m39 & t19; + // m48 -> stack + let m36 = m24 ^ m25; + let m31 = m20 & m23; + let m27 = m20 ^ m21; + let m32 = m27 & m31; + let m29 = m28 & m27; + let m37 = m21 ^ m29; + // m39 -> stack + let m42 = m37 ^ m39; + let m52 = m42 & t15; + // t27 -> stack + // t1 <- stack + let m61 = m42 & t1; + let p0 = m52 ^ m61; + let p16 = m57 ^ m61; + // m57 -> stack + // t20 <- stack + let m60 = m37 & t20; + // p16 -> stack + // t17 <- stack + let m51 = m37 & t17; + let m33 = m27 ^ m25; + let m38 = m32 ^ m33; + let m43 = m37 ^ m38; + let m49 = m43 & t16; + let p6 = m49 ^ m60; + let p13 = m49 ^ m51; + let m58 = m43 & t3; + // t9 <- stack + let m50 = m38 & t9; + // t22 <- stack + let m59 = m38 & t22; + // p6 -> stack + let p1 = m58 ^ m59; + let p7 = p0 ^ p1; + let m34 = m21 & m22; + let m35 = m24 & m34; + let m40 = m35 ^ m36; + let m41 = m38 ^ m40; + let m45 = m42 ^ m41; + // t27 <- stack + let m53 = m45 & t27; + let p8 = m50 ^ m53; + let p23 = p7 ^ p8; + // t4 <- stack + let m62 = m45 & t4; + let p14 = m49 ^ m62; + let s6 = p14 ^ p23; + // t10 <- stack + let m54 = m41 & t10; + let p2 = m54 ^ m62; + let p22 = p2 ^ p7; + let s0 = p13 ^ p22; + let p17 = m58 ^ p2; + let p15 = m54 ^ m59; + // t2 <- stack + let m63 = m41 & t2; + // m39 <- stack + let m44 = m39 ^ m40; + // p17 -> stack + // t6 <- stack + let m46 = m44 & t6; + let p5 = m46 ^ m51; + // p23 -> stack + let p18 = m63 ^ p5; + let p24 = p5 ^ p7; + // m48 <- stack + let p12 = m46 ^ m48; + let s3 = p12 ^ p22; + // t13 <- stack + let m55 = m44 & t13; + let p9 = m55 ^ m63; + // p16 <- stack + let s7 = p9 ^ p16; + // t8 <- stack + let m47 = m40 & t8; + let p3 = m47 ^ m50; + let p19 = p2 ^ p3; + let s5 = p19 ^ p24; + let p11 = p0 ^ p3; + let p26 = p9 ^ p11; + // t23 <- stack + let m56 = m40 & t23; + let p4 = m48 ^ m56; + // p6 <- stack + let p20 = p4 ^ p6; + let p29 = p15 ^ p20; + let s1 = p26 ^ p29; + // m57 <- stack + let p10 = m57 ^ p4; + let p27 = p10 ^ p18; + // p23 <- stack + let s4 = p23 ^ p27; + let p25 = p6 ^ p10; + let p28 = p11 ^ p25; + // p17 <- stack + let s2 = p17 ^ p28; + + state[0] = s7; + state[1] = s6; + state[2] = s5; + state[3] = s4; + state[4] = s3; + state[5] = s2; + state[6] = s1; + state[7] = s0; +} + +/// NOT operations that are omitted in S-box. +#[inline] +pub(crate) fn sub_bytes_nots(state: &mut [W]) { + debug_assert_eq!(state.len(), 8); + state[0] = !state[0]; + state[1] = !state[1]; + state[5] = !state[5]; + state[6] = !state[6]; +} diff --git a/aes/src/backends/fixslice/utils.rs b/aes/src/backends/fixslice/utils.rs new file mode 100644 index 000000000..637488a35 --- /dev/null +++ b/aes/src/backends/fixslice/utils.rs @@ -0,0 +1,119 @@ +use super::{BatchBlocks, State, Word}; + +/// Replicate a single 16-byte input block across all slots of a `Batch`. +/// +/// Used by the key schedules, which conceptually call `bitslice(...)` on the +/// same input block several times to fill the bitsliced state. +pub(crate) fn broadcast(block: &[u8]) -> BatchBlocks { + debug_assert_eq!(block.len(), 16); + let mut out = BatchBlocks::::default(); + for slot in out.iter_mut() { + slot.copy_from_slice(block); + } + out +} + +#[inline] +fn delta_swap_1(a: &mut W, shift: u32, mask: W) { + let t = (*a ^ ((*a) >> shift)) & mask; + *a ^= t ^ (t << shift); +} + +#[inline] +pub(crate) fn delta_swap_2(a: &mut W, b: &mut W, shift: u32, mask: W) { + let t = (*a ^ ((*b) >> shift)) & mask; + *a ^= t; + *b ^= t << shift; +} + +/// Applies ShiftRows once on an AES state (or key). +#[cfg(any(not(aes_backend_soft = "compact"), feature = "hazmat"))] +#[inline] +pub(crate) fn shift_rows_1(state: &mut [W]) { + debug_assert_eq!(state.len(), 8); + for x in state.iter_mut() { + delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x03, 0x0f, 0x0c)); + delta_swap_1(x, W::QUARTER_ROW, W::pack_rows(0x00, 0x33, 0x00, 0x33)); + } +} + +/// Applies ShiftRows twice on an AES state (or key). +#[inline] +pub(crate) fn shift_rows_2(state: &mut [W]) { + debug_assert_eq!(state.len(), 8); + for x in state.iter_mut() { + delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x0f, 0x00, 0x0f)); + } +} + +/// Applies ShiftRows three times on an AES state (or key). +#[inline] +pub(crate) fn shift_rows_3(state: &mut [W]) { + debug_assert_eq!(state.len(), 8); + for x in state.iter_mut() { + delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x0c, 0x0f, 0x03)); + delta_swap_1(x, W::QUARTER_ROW, W::pack_rows(0x00, 0x33, 0x00, 0x33)); + } +} + +#[inline(always)] +pub(crate) fn inv_shift_rows_1(state: &mut [W]) { + shift_rows_3(state); +} + +#[inline(always)] +pub(crate) fn inv_shift_rows_2(state: &mut [W]) { + shift_rows_2(state); +} + +#[cfg(not(aes_backend_soft = "compact"))] +#[inline(always)] +pub(crate) fn inv_shift_rows_3(state: &mut [W]) { + shift_rows_1(state); +} + +/// XOR the columns after the S-box during the key schedule round function. +/// +/// The `idx_xor` parameter refers to the index of the previous round key +/// involved in the XOR computation (should be 8 and 16 for AES-128 and AES-256, +/// respectively). +/// +/// The `idx_ror` parameter refers to the rotation value, which varies between the +/// different key schedules. +pub(crate) fn xor_columns(rkeys: &mut [W], offset: usize, idx_xor: usize, idx_ror: u32) { + for i in 0..8 { + let off_i = offset + i; + let rk = rkeys[off_i - idx_xor] ^ (W::uniform_row(0x03) & rkeys[off_i].ror(idx_ror)); + rkeys[off_i] = rk + ^ (W::uniform_row(0xfc) & (rk << W::QUARTER_ROW)) + ^ (W::uniform_row(0xf0) & (rk << W::HALF_ROW)) + ^ (W::uniform_row(0xc0) & (rk << (3 * W::QUARTER_ROW))); + } +} + +/// Copy 32-bytes within the provided slice to an 8-byte offset. +pub(crate) fn memshift32(buffer: &mut [W], src_offset: usize) { + debug_assert_eq!(src_offset % 8, 0); + + let dst_offset = src_offset + 8; + debug_assert!(dst_offset + 8 <= buffer.len()); + + for i in (0..8).rev() { + buffer[dst_offset + i] = buffer[src_offset + i]; + } +} + +/// XOR the round key into the internal state. The round keys are expected +/// to be pre-computed and packed in the fixsliced representation. +#[inline] +pub(crate) fn add_round_key(state: &mut State, rkey: &[W]) { + debug_assert_eq!(rkey.len(), 8); + for (a, b) in state.iter_mut().zip(rkey) { + *a ^= *b; + } +} + +#[inline(always)] +pub(crate) fn add_round_constant_bit(state: &mut [W], bit: usize) { + state[bit] ^= W::pack_rows(0x00, 0xc0, 0x00, 0x00); +} diff --git a/aes/src/backends/fixslice/word.rs b/aes/src/backends/fixslice/word.rs new file mode 100644 index 000000000..e24abc817 --- /dev/null +++ b/aes/src/backends/fixslice/word.rs @@ -0,0 +1,324 @@ +use crate::Block; +use cipher::{ + Array, + array::ArraySize, + consts::{U2, U4}, +}; +use core::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Shl, Shr}; + +/// Width-abstracted machine word holding one row of a bitsliced AES state. +pub(crate) trait Word: + Sized + + Copy + + Default + + 'static + + BitAnd + + BitAndAssign + + BitOr + + BitOrAssign + + BitXor + + BitXorAssign + + Not + + Shl + + Shr +{ + /// Number of 128-bit blocks bitsliced together in one state. + type Blocks: ArraySize; + + /// Width in bits of one row of the bitsliced state (8 for `u32`, 16 for `u64`). + const ROW_BITS: u32 = (size_of::() * 2) as u32; + + /// Half of `ROW_BITS`. + const HALF_ROW: u32 = Self::ROW_BITS / 2; + /// Quarter of `ROW_BITS`. + const QUARTER_ROW: u32 = Self::ROW_BITS / 4; + + /// Distance in bits to rotate a state row by `(rows, cols)` positions. + #[inline(always)] + fn ror_distance(rows: u32, cols: u32) -> u32 { + rows * Self::ROW_BITS + cols * Self::QUARTER_ROW + } + + /// Rotate right by `n` bits. + fn ror(self, n: u32) -> Self; + + /// Pack the same byte across all 4 rows of the word. + fn uniform_row(b: u8) -> Self; + + /// Place one byte at each of the 4 row positions of the word (row 0 = LSB). + fn pack_rows(r0: u8, r1: u8, r2: u8, r3: u8) -> Self; + + /// Replicate byte `b` across every byte of the word. + fn byte_repeat(b: u8) -> Self; + + /// Pack `Self::Blocks` input blocks into a bitsliced 8-row state slice. + fn bitslice(output: &mut [Self], input: &Array); + + /// Unpack a bitsliced 8-row state slice into `Self::Blocks` output blocks. + fn inv_bitslice(input: &[Self]) -> Array; +} + +impl Word for u32 { + type Blocks = U2; + + #[inline(always)] + fn ror(self, n: u32) -> u32 { + self.rotate_right(n) + } + + #[inline(always)] + fn uniform_row(b: u8) -> u32 { + (b as u32) * 0x01010101 + } + + #[inline(always)] + fn pack_rows(r0: u8, r1: u8, r2: u8, r3: u8) -> u32 { + (r0 as u32) | ((r1 as u32) << 8) | ((r2 as u32) << 16) | ((r3 as u32) << 24) + } + + #[inline(always)] + fn byte_repeat(b: u8) -> u32 { + (b as u32) * 0x01010101 + } + + /// Bitslice two 128-bit input blocks into a 256-bit internal state. + fn bitslice(output: &mut [u32], input: &Array) { + debug_assert_eq!(output.len(), 8); + let input0 = input[0].as_slice(); + let input1 = input[1].as_slice(); + + // Bitslicing is a bit index manipulation. 256 bits of data means each bit is positioned at + // an 8-bit index. AES data is 2 blocks, each one a 4x4 column-major matrix of bytes, so the + // index is initially ([b]lock, [c]olumn, [r]ow, [p]osition): + // b0 c1 c0 r1 r0 p2 p1 p0 + // + // The desired bitsliced data groups first by bit position, then row, column, block: + // p2 p1 p0 r1 r0 c1 c0 b0 + + // Interleave the columns on input (note the order of input) + // b0 c1 c0 __ __ __ __ __ => c1 c0 b0 __ __ __ __ __ + let mut t = [ + u32::from_le_bytes(input0[0x00..0x04].try_into().unwrap()), + u32::from_le_bytes(input1[0x00..0x04].try_into().unwrap()), + u32::from_le_bytes(input0[0x04..0x08].try_into().unwrap()), + u32::from_le_bytes(input1[0x04..0x08].try_into().unwrap()), + u32::from_le_bytes(input0[0x08..0x0c].try_into().unwrap()), + u32::from_le_bytes(input1[0x08..0x0c].try_into().unwrap()), + u32::from_le_bytes(input0[0x0c..0x10].try_into().unwrap()), + u32::from_le_bytes(input1[0x0c..0x10].try_into().unwrap()), + ]; + + bitslice_swaps(&mut t); + + // Final bitsliced bit index, as desired: + // p2 p1 p0 r1 r0 c1 c0 b0 + output[..8].copy_from_slice(&t); + } + + /// Un-bitslice a 256-bit internal state into two 128-bit blocks. + fn inv_bitslice(input: &[u32]) -> Array { + debug_assert_eq!(input.len(), 8); + + // Unbitslicing is a bit index manipulation. 256 bits of data means each bit is positioned + // at an 8-bit index. AES data is 2 blocks, each one a 4x4 column-major matrix of bytes, so + // the desired index for the output is ([b]lock, [c]olumn, [r]ow, [p]osition): + // b0 c1 c0 r1 r0 p2 p1 p0 + // + // The initially bitsliced data groups first by bit position, then row, column, block: + // p2 p1 p0 r1 r0 c1 c0 b0 + + let mut t = [ + input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7], + ]; + + bitslice_swaps(&mut t); + + let mut output = Array::::default(); + // De-interleave the columns on output (note the order of output) + // c1 c0 b0 __ __ __ __ __ => b0 c1 c0 __ __ __ __ __ + output[0][0x00..0x04].copy_from_slice(&t[0].to_le_bytes()); + output[0][0x04..0x08].copy_from_slice(&t[2].to_le_bytes()); + output[0][0x08..0x0c].copy_from_slice(&t[4].to_le_bytes()); + output[0][0x0c..0x10].copy_from_slice(&t[6].to_le_bytes()); + output[1][0x00..0x04].copy_from_slice(&t[1].to_le_bytes()); + output[1][0x04..0x08].copy_from_slice(&t[3].to_le_bytes()); + output[1][0x08..0x0c].copy_from_slice(&t[5].to_le_bytes()); + output[1][0x0c..0x10].copy_from_slice(&t[7].to_le_bytes()); + + // Final AES bit index, as desired: + // b0 c1 c0 r1 r0 p2 p1 p0 + output + } +} + +/// Expand an 8-bit row pattern to a 16-bit row pattern by doubling each bit: +/// input bit `i` becomes output bits `2i` and `2i+1`. Branchless SWAR so LLVM +/// folds it to a single 16-bit immediate when `b` is a constant. +#[inline(always)] +const fn double_bits(b: u8) -> u16 { + let x = b as u16; + // Spread the 8 bits of x to even positions 0,2,4,6,8,10,12,14. + let x = (x | (x << 4)) & 0x0f0f; + let x = (x | (x << 2)) & 0x3333; + let x = (x | (x << 1)) & 0x5555; + // Duplicate each spread bit to its adjacent odd position. + x | (x << 1) +} + +impl Word for u64 { + type Blocks = U4; + + #[inline(always)] + fn ror(self, n: u32) -> u64 { + self.rotate_right(n) + } + + #[inline(always)] + fn uniform_row(b: u8) -> u64 { + (double_bits(b) as u64) * 0x0001_0001_0001_0001 + } + + #[inline(always)] + fn pack_rows(r0: u8, r1: u8, r2: u8, r3: u8) -> u64 { + (double_bits(r0) as u64) + | ((double_bits(r1) as u64) << 16) + | ((double_bits(r2) as u64) << 32) + | ((double_bits(r3) as u64) << 48) + } + + #[inline(always)] + fn byte_repeat(b: u8) -> u64 { + (b as u64) * 0x0101010101010101 + } + + /// Bitslice four 128-bit input blocks into a 512-bit internal state. + fn bitslice(output: &mut [u64], input: &Array) { + debug_assert_eq!(output.len(), 8); + + // Bitslicing is a bit index manipulation. 512 bits of data means each bit is positioned at + // a 9-bit index. AES data is 4 blocks, each one a 4x4 column-major matrix of bytes, so the + // index is initially ([b]lock, [c]olumn, [r]ow, [p]osition): + // b1 b0 c1 c0 r1 r0 p2 p1 p0 + // + // The desired bitsliced data groups first by bit position, then row, column, block: + // p2 p1 p0 r1 r0 c1 c0 b1 b0 + + #[rustfmt::skip] + fn read_reordered(input: &[u8]) -> u64 { + (u64::from(input[0x0]) ) | + (u64::from(input[0x1]) << 0x10) | + (u64::from(input[0x2]) << 0x20) | + (u64::from(input[0x3]) << 0x30) | + (u64::from(input[0x8]) << 0x08) | + (u64::from(input[0x9]) << 0x18) | + (u64::from(input[0xa]) << 0x28) | + (u64::from(input[0xb]) << 0x38) + } + + // Reorder each block's bytes on input + // __ __ c1 c0 r1 r0 __ __ __ => __ __ c0 r1 r0 c1 __ __ __ + // Reorder by relabeling (note the order of input) + // b1 b0 c0 __ __ __ __ __ __ => c0 b1 b0 __ __ __ __ __ __ + let mut t = [ + read_reordered(&input[0][0x00..0x0c]), + read_reordered(&input[1][0x00..0x0c]), + read_reordered(&input[2][0x00..0x0c]), + read_reordered(&input[3][0x00..0x0c]), + read_reordered(&input[0][0x04..0x10]), + read_reordered(&input[1][0x04..0x10]), + read_reordered(&input[2][0x04..0x10]), + read_reordered(&input[3][0x04..0x10]), + ]; + + bitslice_swaps(&mut t); + + // Final bitsliced bit index, as desired: + // p2 p1 p0 r1 r0 c1 c0 b1 b0 + output[..8].copy_from_slice(&t); + } + + /// Un-bitslice a 512-bit internal state into four 128-bit blocks. + fn inv_bitslice(input: &[u64]) -> Array { + debug_assert_eq!(input.len(), 8); + + // Unbitslicing is a bit index manipulation. 512 bits of data means each bit is positioned + // at a 9-bit index. AES data is 4 blocks, each one a 4x4 column-major matrix of bytes, so + // the desired index for the output is ([b]lock, [c]olumn, [r]ow, [p]osition): + // b1 b0 c1 c0 r1 r0 p2 p1 p0 + // + // The initially bitsliced data groups first by bit position, then row, column, block: + // p2 p1 p0 r1 r0 c1 c0 b1 b0 + + let mut t = [ + input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7], + ]; + + bitslice_swaps(&mut t); + + #[rustfmt::skip] + fn write_reordered(columns: u64, output: &mut [u8]) { + output[0x0] = (columns ) as u8; + output[0x1] = (columns >> 0x10) as u8; + output[0x2] = (columns >> 0x20) as u8; + output[0x3] = (columns >> 0x30) as u8; + output[0x8] = (columns >> 0x08) as u8; + output[0x9] = (columns >> 0x18) as u8; + output[0xa] = (columns >> 0x28) as u8; + output[0xb] = (columns >> 0x38) as u8; + } + + let mut output = Array::::default(); + // Reorder by relabeling (note the order of output) + // c0 b1 b0 __ __ __ __ __ __ => b1 b0 c0 __ __ __ __ __ __ + // Reorder each block's bytes on output + // __ __ c0 r1 r0 c1 __ __ __ => __ __ c1 c0 r1 r0 __ __ __ + write_reordered(t[0], &mut output[0][0x00..0x0c]); + write_reordered(t[4], &mut output[0][0x04..0x10]); + write_reordered(t[1], &mut output[1][0x00..0x0c]); + write_reordered(t[5], &mut output[1][0x04..0x10]); + write_reordered(t[2], &mut output[2][0x00..0x0c]); + write_reordered(t[6], &mut output[2][0x04..0x10]); + write_reordered(t[3], &mut output[3][0x00..0x0c]); + write_reordered(t[7], &mut output[3][0x04..0x10]); + + // Final AES bit index, as desired: + // b1 b0 c1 c0 r1 r0 p2 p1 p0 + output + } +} + +/// Width-generic delta-swap pipeline shared by `bitslice` and `inv_bitslice` +/// across every `Word` impl. The same three-pass sequence inverts itself, so +/// `bitslice` and `inv_bitslice` invoke it identically. +/// +/// The diagrams below describe the `u32` case (8-bit rows); for `u64` each +/// bit position widens by one, but the swap structure is unchanged. +#[inline(always)] +fn bitslice_swaps(t: &mut [W; 8]) { + use super::utils::delta_swap_2; + let [t0, t1, t2, t3, t4, t5, t6, t7] = t; + + // Bit Index Swap 5 <-> 0: + // __ __ b0 __ __ __ __ p0 => __ __ p0 __ __ __ __ b0 + let m0 = W::byte_repeat(0x55); + delta_swap_2(t1, t0, 1, m0); + delta_swap_2(t3, t2, 1, m0); + delta_swap_2(t5, t4, 1, m0); + delta_swap_2(t7, t6, 1, m0); + + // Bit Index Swap 6 <-> 1: + // __ c0 __ __ __ __ p1 __ => __ p1 __ __ __ __ c0 __ + let m1 = W::byte_repeat(0x33); + delta_swap_2(t2, t0, 2, m1); + delta_swap_2(t3, t1, 2, m1); + delta_swap_2(t6, t4, 2, m1); + delta_swap_2(t7, t5, 2, m1); + + // Bit Index Swap 7 <-> 2: + // c1 __ __ __ __ p2 __ __ => p2 __ __ __ __ c1 __ __ + let m2 = W::byte_repeat(0x0f); + delta_swap_2(t4, t0, 4, m2); + delta_swap_2(t5, t1, 4, m2); + delta_swap_2(t6, t2, 4, m2); + delta_swap_2(t7, t3, 4, m2); +} diff --git a/aes/src/backends/soft.rs b/aes/src/backends/soft.rs new file mode 100644 index 000000000..07c35a654 --- /dev/null +++ b/aes/src/backends/soft.rs @@ -0,0 +1,97 @@ +#![deny(unsafe_code)] +use crate::Block; +use cipher::{ + BlockCipherDecBackend, BlockCipherEncBackend, BlockSizeUser, ParBlocks, ParBlocksSizeUser, + consts::U16, inout::InOut, +}; + +#[path = "fixslice/mod.rs"] +pub(crate) mod fixslice; + +#[cfg(feature = "hazmat")] +pub(crate) use fixslice::hazmat; + +use fixslice::{BatchBlocks, NativeBatchSize, NativeWord}; + +macro_rules! impl_backend { + ( + name = $name:tt, + key_size = $key_size:literal, + module = $module:ident, + doc = $doc:expr, + ) => { + #[doc=$doc] + #[doc = "block cipher"] + #[derive(Clone, Copy)] + pub(crate) struct $name { + keys: fixslice::$module::RoundKeys, + } + + impl $name { + #[inline] + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let keys = fixslice::$module::key_schedule(key); + Self { keys } + } + } + + impl BlockSizeUser for $name { + type BlockSize = U16; + } + + impl ParBlocksSizeUser for $name { + type ParBlocksSize = NativeBatchSize; + } + + impl BlockCipherEncBackend for $name { + #[inline(always)] + fn encrypt_block(&self, mut block: InOut<'_, '_, Block>) { + let mut blocks = BatchBlocks::::default(); + blocks[0] = block.clone_in().into(); + let res = fixslice::$module::encrypt(&self.keys, &blocks); + *block.get_out() = res[0].into(); + } + + #[inline(always)] + fn encrypt_par_blocks(&self, mut blocks: InOut<'_, '_, ParBlocks>) { + let res = fixslice::$module::encrypt(&self.keys, blocks.get_in()); + *blocks.get_out() = res; + } + } + + impl BlockCipherDecBackend for $name { + #[inline(always)] + fn decrypt_block(&self, mut block: InOut<'_, '_, Block>) { + let mut blocks = BatchBlocks::::default(); + blocks[0] = block.clone_in(); + let res = fixslice::$module::decrypt(&self.keys, &blocks); + *block.get_out() = res[0]; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, mut blocks: InOut<'_, '_, ParBlocks>) { + let res = fixslice::$module::decrypt(&self.keys, blocks.get_in()); + *blocks.get_out() = res; + } + } + }; +} + +impl_backend!( + name = Aes128, + key_size = 16, + module = aes128, + doc = "AES-128", +); +impl_backend!( + name = Aes192, + key_size = 24, + module = aes192, + doc = "AES-192", +); +impl_backend!( + name = Aes256, + key_size = 32, + module = aes256, + doc = "AES-256", +); diff --git a/aes/src/backends/x86_aes.rs b/aes/src/backends/x86_aes.rs new file mode 100644 index 000000000..ac0290b03 --- /dev/null +++ b/aes/src/backends/x86_aes.rs @@ -0,0 +1,228 @@ +use cipher::{ + Block, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherEncBackend, + BlockCipherEncClosure, BlockSizeUser, ParBlocks, ParBlocksSizeUser, + consts::{U8, U16}, + inout::InOut, +}; + +mod encdec; +mod expand; + +#[cfg(feature = "hazmat")] +pub(crate) mod hazmat; + +#[cfg(any(aes_backend = "avx512", aes_backend = "avx256"))] +pub(crate) use encdec::{decrypt, encrypt}; +pub(crate) use expand::RoundKeys; + +pub(crate) type Aes128 = Aes<11>; +pub(crate) type Aes192 = Aes<13>; +pub(crate) type Aes256 = Aes<15>; + +pub(crate) type Aes128Enc = AesEnc<11>; +pub(crate) type Aes192Enc = AesEnc<13>; +pub(crate) type Aes256Enc = AesEnc<15>; + +pub(crate) type Aes128Dec = AesDec<11>; +pub(crate) type Aes192Dec = AesDec<13>; +pub(crate) type Aes256Dec = AesDec<15>; + +type ParBlocksSize = U8; + +#[derive(Clone, Copy)] +pub(crate) struct Aes { + pub(crate) enc_rk: RoundKeys, + pub(crate) dec_rk: RoundKeys, +} + +impl Aes { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + f.call(self); + } + + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + f.call(self); + } +} + +impl BlockSizeUser for Aes { + type BlockSize = U16; +} + +impl ParBlocksSizeUser for Aes { + type ParBlocksSize = ParBlocksSize; +} + +impl BlockCipherEncBackend for Aes { + #[inline(always)] + fn encrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt(&self.enc_rk, block) }; + } + + #[inline(always)] + fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + } +} + +impl BlockCipherDecBackend for Aes { + #[inline(always)] + fn decrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt(&self.dec_rk, block) }; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt_par(&self.dec_rk, blocks) }; + } +} + +#[derive(Clone, Copy)] +pub(crate) struct AesEnc { + pub(crate) enc_rk: RoundKeys, +} + +impl AesEnc { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn as_encdec(&self) -> Aes { + let enc_rk = self.enc_rk; + // SAFETY: the is method marked with `#[target_feature(enable = "aes")]` + let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + Aes { enc_rk, dec_rk } + } + + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn as_dec(&self) -> AesDec { + // SAFETY: the is method marked with `#[target_feature(enable = "aes")]` + let dec_rk = unsafe { expand::inv_expanded_keys(&self.enc_rk) }; + AesDec { dec_rk } + } + + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + f.call(self) + } +} + +impl BlockSizeUser for AesEnc { + type BlockSize = U16; +} + +impl ParBlocksSizeUser for AesEnc { + type ParBlocksSize = ParBlocksSize; +} + +impl BlockCipherEncBackend for AesEnc { + #[inline(always)] + fn encrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt(&self.enc_rk, block) }; + } + + #[inline(always)] + fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + } +} + +#[derive(Clone, Copy)] +pub(crate) struct AesDec { + pub(crate) dec_rk: RoundKeys, +} + +impl AesDec { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + f.call(self); + } +} + +impl BlockSizeUser for AesDec { + type BlockSize = U16; +} + +impl ParBlocksSizeUser for AesDec { + type ParBlocksSize = ParBlocksSize; +} + +impl BlockCipherDecBackend for AesDec { + #[inline(always)] + fn decrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt(&self.dec_rk, block) }; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "aes")]` + unsafe { encdec::decrypt_par(&self.dec_rk, blocks) }; + } +} + +macro_rules! impl_key_init { + ($name:ty, $name_enc:ty, $name_dec:ty, $key_size:literal, $expand_fn:ident) => { + impl $name { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = unsafe { expand::$expand_fn(key) }; + let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + Self { enc_rk, dec_rk } + } + } + + impl $name_enc { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { + // SAFETY: the is method marked with `#[target_feature(enable = "aes")]` + let enc_rk = unsafe { expand::$expand_fn(key) }; + Self { enc_rk } + } + } + + impl $name_dec { + #[inline] + #[target_feature(enable = "aes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = unsafe { expand::$expand_fn(key) }; + let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + Self { dec_rk } + } + } + }; +} + +impl_key_init!(Aes128, Aes128Enc, Aes128Dec, 16, aes128_expand_key); +impl_key_init!(Aes192, Aes192Enc, Aes192Dec, 24, aes192_expand_key); +impl_key_init!(Aes256, Aes256Enc, Aes256Dec, 32, aes256_expand_key); diff --git a/aes/src/ni/encdec.rs b/aes/src/backends/x86_aes/encdec.rs similarity index 61% rename from aes/src/ni/encdec.rs rename to aes/src/backends/x86_aes/encdec.rs index 12db0da26..fa4a245e0 100644 --- a/aes/src/ni/encdec.rs +++ b/aes/src/backends/x86_aes/encdec.rs @@ -1,55 +1,56 @@ #![allow(unsafe_op_in_unsafe_fn)] -use super::arch::*; +use super::RoundKeys; use crate::Block; use cipher::{ array::{Array, ArraySize}, inout::InOut, }; +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn encrypt( - keys: &[__m128i; KEYS], - block: InOut<'_, '_, Block>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); +pub(crate) unsafe fn encrypt(keys: &RoundKeys, block: InOut<'_, '_, Block>) { + const { assert!(matches!(RK, 11 | 13 | 15)) } let (block_in, block_out) = block.into_raw(); let mut b = _mm_loadu_si128(block_in.cast()); b = _mm_xor_si128(b, keys[0]); - for &key in &keys[1..KEYS - 1] { + for &key in &keys[1..RK - 1] { b = _mm_aesenc_si128(b, key); } - b = _mm_aesenclast_si128(b, keys[KEYS - 1]); + b = _mm_aesenclast_si128(b, keys[RK - 1]); _mm_storeu_si128(block_out.cast(), b); } +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn decrypt( - keys: &[__m128i; KEYS], - block: InOut<'_, '_, Block>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); +pub(crate) unsafe fn decrypt(keys: &RoundKeys, block: InOut<'_, '_, Block>) { + const { assert!(matches!(RK, 11 | 13 | 15)) } let (block_in, block_out) = block.into_raw(); let mut b = _mm_loadu_si128(block_in.cast()); b = _mm_xor_si128(b, keys[0]); - for &key in &keys[1..KEYS - 1] { + for &key in &keys[1..RK - 1] { b = _mm_aesdec_si128(b, key); } - b = _mm_aesdeclast_si128(b, keys[KEYS - 1]); + b = _mm_aesdeclast_si128(b, keys[RK - 1]); _mm_storeu_si128(block_out.cast(), b); } +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn encrypt_par( - keys: &[__m128i; KEYS], - blocks: InOut<'_, '_, Array>, +pub(super) unsafe fn encrypt_par( + keys: &RoundKeys, + mut blocks: InOut<'_, '_, Array>, ) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); + const { assert!(matches!(RK, 11 | 13 | 15)) } - let (blocks_in, blocks_out) = blocks.into_raw(); - let mut b = load(blocks_in); + let mut b = load(blocks.get_in()); // Loop over keys is intentionally not used here to force inlining xor(&mut b, keys[0]); @@ -62,27 +63,27 @@ pub(super) unsafe fn encrypt_par( aesenc(&mut b, keys[7]); aesenc(&mut b, keys[8]); aesenc(&mut b, keys[9]); - if KEYS >= 13 { + if RK >= 13 { aesenc(&mut b, keys[10]); aesenc(&mut b, keys[11]); } - if KEYS == 15 { + if RK == 15 { aesenc(&mut b, keys[12]); aesenc(&mut b, keys[13]); } - aesenclast(&mut b, keys[KEYS - 1]); - store(blocks_out, b); + aesenclast(&mut b, keys[RK - 1]); + store(blocks.get_out(), b); } +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn decrypt_par( - keys: &[__m128i; KEYS], - blocks: InOut<'_, '_, Array>, +pub(super) unsafe fn decrypt_par( + keys: &RoundKeys, + mut blocks: InOut<'_, '_, Array>, ) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); + const { assert!(matches!(RK, 11 | 13 | 15)) }; - let (blocks_in, blocks_out) = blocks.into_raw(); - let mut b = load(blocks_in); + let mut b = load(blocks.get_in()); // Loop over keys is intentionally not used here to force inlining xor(&mut b, keys[0]); @@ -95,33 +96,29 @@ pub(super) unsafe fn decrypt_par( aesdec(&mut b, keys[7]); aesdec(&mut b, keys[8]); aesdec(&mut b, keys[9]); - if KEYS >= 13 { + if RK >= 13 { aesdec(&mut b, keys[10]); aesdec(&mut b, keys[11]); } - if KEYS == 15 { + if RK == 15 { aesdec(&mut b, keys[12]); aesdec(&mut b, keys[13]); } - aesdeclast(&mut b, keys[KEYS - 1]); - store(blocks_out, b); + aesdeclast(&mut b, keys[RK - 1]); + store(blocks.get_out(), b); } #[target_feature(enable = "sse2")] -pub(crate) unsafe fn load(blocks: *const Array) -> Array<__m128i, N> { - let p = blocks.cast::<__m128i>(); - let mut res: Array<__m128i, N> = core::mem::zeroed(); - for i in 0..N::USIZE { - res[i] = _mm_loadu_si128(p.add(i)); - } - res +pub(crate) unsafe fn load(blocks: &Array) -> Array<__m128i, N> { + let p: *const __m128i = blocks.as_ptr().cast(); + Array::from_fn(|i| unsafe { _mm_loadu_si128(p.add(i)) }) } #[target_feature(enable = "sse2")] -pub(crate) unsafe fn store(blocks: *mut Array, b: Array<__m128i, N>) { - let p = blocks.cast::<__m128i>(); - for i in 0..N::USIZE { - _mm_storeu_si128(p.add(i), b[i]); +pub(crate) unsafe fn store(dst: &mut Array, blocks: Array<__m128i, N>) { + let p: *mut __m128i = dst.as_mut_ptr().cast(); + for (i, block) in blocks.into_iter().enumerate() { + unsafe { _mm_storeu_si128(p.add(i), block) } } } diff --git a/aes/src/ni/expand.rs b/aes/src/backends/x86_aes/expand.rs similarity index 71% rename from aes/src/ni/expand.rs rename to aes/src/backends/x86_aes/expand.rs index 325c4c349..3f07f43f4 100644 --- a/aes/src/ni/expand.rs +++ b/aes/src/backends/x86_aes/expand.rs @@ -1,20 +1,21 @@ #![allow(unsafe_op_in_unsafe_fn)] -use super::arch::*; -use core::mem::{transmute, zeroed}; +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; -pub(super) type Aes128RoundKeys = [__m128i; 11]; -pub(super) type Aes192RoundKeys = [__m128i; 13]; -pub(super) type Aes256RoundKeys = [__m128i; 15]; +pub(crate) type RoundKeys = [__m128i; RK]; +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> Aes128RoundKeys { - unsafe fn expand_round(keys: &mut Aes128RoundKeys, pos: usize) { +pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> RoundKeys<11> { + unsafe fn expand_round(keys: &mut RoundKeys<11>, pos: usize) { let mut t1 = keys[pos - 1]; let mut t2; let mut t3; - t2 = _mm_aeskeygenassist_si128(t1, RK); + t2 = _mm_aeskeygenassist_si128(t1, R); t2 = _mm_shuffle_epi32(t2, 0xff); t3 = _mm_slli_si128(t1, 0x4); t1 = _mm_xor_si128(t1, t3); @@ -27,7 +28,7 @@ pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> Aes128RoundKeys { keys[pos] = t1; } - let mut keys: Aes128RoundKeys = zeroed(); + let mut keys = [_mm_setzero_si128(); 11]; let k = _mm_loadu_si128(key.as_ptr().cast()); keys[0] = k; @@ -46,19 +47,19 @@ pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> Aes128RoundKeys { keys } +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> Aes192RoundKeys { - unsafe fn shuffle(a: __m128i, b: __m128i, i: usize) -> __m128i { - let a: [u64; 2] = transmute(a); - let b: [u64; 2] = transmute(b); - transmute([a[i], b[0]]) +pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> RoundKeys<13> { + unsafe fn unpack_hilo(a: __m128i, b: __m128i) -> __m128i { + let a = _mm_shuffle_epi32(a, 0b01_00_11_10); + _mm_unpacklo_epi64(a, b) } #[target_feature(enable = "aes")] - unsafe fn expand_round(mut t1: __m128i, mut t3: __m128i) -> (__m128i, __m128i) { + unsafe fn expand_round(mut t1: __m128i, mut t3: __m128i) -> (__m128i, __m128i) { let (mut t2, mut t4); - t2 = _mm_aeskeygenassist_si128(t3, RK); + t2 = _mm_aeskeygenassist_si128(t3, R); t2 = _mm_shuffle_epi32(t2, 0x55); t4 = _mm_slli_si128(t1, 0x4); t1 = _mm_xor_si128(t1, t4); @@ -75,7 +76,7 @@ pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> Aes192RoundKeys { (t1, t3) } - let mut keys: Aes192RoundKeys = zeroed(); + let mut keys = [_mm_setzero_si128(); 13]; // We are being extra pedantic here to remove out-of-bound access. // This should be optimized into movups, movsd sequence. let (k0, k1l) = { @@ -90,15 +91,15 @@ pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> Aes192RoundKeys { keys[0] = k0; let (k1_2, k2r) = expand_round::<0x01>(k0, k1l); - keys[1] = shuffle(k1l, k1_2, 0); - keys[2] = shuffle(k1_2, k2r, 1); + keys[1] = _mm_unpacklo_epi64(k1l, k1_2); + keys[2] = unpack_hilo(k1_2, k2r); let (k3, k4l) = expand_round::<0x02>(k1_2, k2r); keys[3] = k3; let (k4_5, k5r) = expand_round::<0x04>(k3, k4l); - let k4 = shuffle(k4l, k4_5, 0); - let k5 = shuffle(k4_5, k5r, 1); + let k4 = _mm_unpacklo_epi64(k4l, k4_5); + let k5 = unpack_hilo(k4_5, k5r); keys[4] = k4; keys[5] = k5; @@ -106,15 +107,15 @@ pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> Aes192RoundKeys { keys[6] = k6; let (k7_8, k8r) = expand_round::<0x10>(k6, k7l); - keys[7] = shuffle(k7l, k7_8, 0); - keys[8] = shuffle(k7_8, k8r, 1); + keys[7] = _mm_unpacklo_epi64(k7l, k7_8); + keys[8] = unpack_hilo(k7_8, k8r); let (k9, k10l) = expand_round::<0x20>(k7_8, k8r); keys[9] = k9; let (k10_11, k11r) = expand_round::<0x40>(k9, k10l); - keys[10] = shuffle(k10l, k10_11, 0); - keys[11] = shuffle(k10_11, k11r, 1); + keys[10] = _mm_unpacklo_epi64(k10l, k10_11); + keys[11] = unpack_hilo(k10_11, k11r); let (k12, _) = expand_round::<0x80>(k10_11, k11r); keys[12] = k12; @@ -122,15 +123,16 @@ pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> Aes192RoundKeys { keys } +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> Aes256RoundKeys { - unsafe fn expand_round(keys: &mut Aes256RoundKeys, pos: usize) { +pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> { + unsafe fn expand_round(keys: &mut RoundKeys<15>, pos: usize) { let mut t1 = keys[pos - 2]; let mut t2; let mut t3 = keys[pos - 1]; let mut t4; - t2 = _mm_aeskeygenassist_si128(t3, RK); + t2 = _mm_aeskeygenassist_si128(t3, R); t2 = _mm_shuffle_epi32(t2, 0xff); t4 = _mm_slli_si128(t1, 0x4); t1 = _mm_xor_si128(t1, t4); @@ -155,13 +157,13 @@ pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> Aes256RoundKeys { keys[pos + 1] = t3; } - unsafe fn expand_round_last(keys: &mut Aes256RoundKeys, pos: usize) { + unsafe fn expand_round_last(keys: &mut RoundKeys<15>, pos: usize) { let mut t1 = keys[pos - 2]; let mut t2; let t3 = keys[pos - 1]; let mut t4; - t2 = _mm_aeskeygenassist_si128(t3, RK); + t2 = _mm_aeskeygenassist_si128(t3, R); t2 = _mm_shuffle_epi32(t2, 0xff); t4 = _mm_slli_si128(t1, 0x4); t1 = _mm_xor_si128(t1, t4); @@ -174,7 +176,7 @@ pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> Aes256RoundKeys { keys[pos] = t1; } - let mut keys: Aes256RoundKeys = zeroed(); + let mut keys = [_mm_setzero_si128(); 15]; let kp = key.as_ptr().cast::<__m128i>(); keys[0] = _mm_loadu_si128(kp); @@ -192,9 +194,10 @@ pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> Aes256RoundKeys { keys } +#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn inv_keys(keys: &[__m128i; N]) -> [__m128i; N] { - let mut inv_keys: [__m128i; N] = zeroed(); +pub(super) unsafe fn inv_expanded_keys(keys: &[__m128i; N]) -> [__m128i; N] { + let mut inv_keys: [__m128i; N] = [_mm_setzero_si128(); N]; inv_keys[0] = keys[N - 1]; for i in 1..N - 1 { inv_keys[i] = _mm_aesimc_si128(keys[N - 1 - i]); diff --git a/aes/src/x86/ni/hazmat.rs b/aes/src/backends/x86_aes/hazmat.rs similarity index 96% rename from aes/src/x86/ni/hazmat.rs rename to aes/src/backends/x86_aes/hazmat.rs index 24a365a5c..b2f8041bc 100644 --- a/aes/src/x86/ni/hazmat.rs +++ b/aes/src/backends/x86_aes/hazmat.rs @@ -6,9 +6,13 @@ #![allow(unsafe_op_in_unsafe_fn)] use crate::hazmat::{Block, Block8}; -use crate::x86::arch::*; use cipher::array::{Array, ArraySize}; +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + #[target_feature(enable = "sse2")] pub(crate) unsafe fn load(blocks: *const Array) -> Array<__m128i, N> { let p = blocks.cast::<__m128i>(); diff --git a/aes/src/backends/x86_vaes256.rs b/aes/src/backends/x86_vaes256.rs new file mode 100644 index 000000000..ae2dbcc9e --- /dev/null +++ b/aes/src/backends/x86_vaes256.rs @@ -0,0 +1,89 @@ +use super::x86_aes::RoundKeys; +use cipher::{ + Block, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherEncBackend, + BlockCipherEncClosure, BlockSizeUser, ParBlocks, ParBlocksSizeUser, + consts::{U16, U30}, + inout::InOut, +}; + +mod encdec; + +pub(crate) type Aes128<'a> = Aes<'a, 11>; +pub(crate) type Aes192<'a> = Aes<'a, 13>; +pub(crate) type Aes256<'a> = Aes<'a, 15>; + +#[derive(Clone, Copy)] +pub(crate) struct Aes<'a, const RK: usize> { + rk: &'a RoundKeys, + rk2: encdec::RoundKeys2, +} + +impl<'a, const RK: usize> Aes<'a, RK> { + #[inline] + #[target_feature(enable = "vaes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn encrypt( + rk: &'a RoundKeys, + f: impl BlockCipherEncClosure, + ) { + let rk2 = unsafe { encdec::broadcast_keys(rk) }; + let backend = Self { rk, rk2 }; + f.call(&backend) + } + + #[inline] + #[target_feature(enable = "vaes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn decrypt( + rk: &'a RoundKeys, + f: impl BlockCipherDecClosure, + ) { + let rk2 = unsafe { encdec::broadcast_keys(rk) }; + let backend = Self { rk, rk2 }; + f.call(&backend) + } +} + +impl BlockSizeUser for Aes<'_, RK> { + type BlockSize = U16; +} + +// Block size of 30 is chosen based on AVX2's 16 YMM registers. +// +// - 1 register holds round key +// - 15 registers hold 2 data blocks +impl ParBlocksSizeUser for Aes<'_, RK> { + type ParBlocksSize = U30; +} + +impl BlockCipherEncBackend for Aes<'_, RK> { + #[inline(always)] + fn encrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "vaes")]` + unsafe { super::x86_aes::encrypt(&self.rk, block) }; + } + + #[inline(always)] + fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "vaes")]` + unsafe { encdec::encrypt_par(&self.rk2, blocks) }; + } +} + +impl BlockCipherDecBackend for Aes<'_, RK> { + #[inline(always)] + fn decrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "vaes")]` + unsafe { super::x86_aes::decrypt(&self.rk, block) }; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "vaes")]` + unsafe { encdec::decrypt_par(&self.rk2, blocks) }; + } +} diff --git a/aes/src/backends/x86_vaes256/encdec.rs b/aes/src/backends/x86_vaes256/encdec.rs new file mode 100644 index 000000000..93280a520 --- /dev/null +++ b/aes/src/backends/x86_vaes256/encdec.rs @@ -0,0 +1,109 @@ +use super::RoundKeys; +use crate::Block; +use cipher::{Array, array::ArraySize, consts::U2, inout::InOut, typenum::Quot}; +use core::ops::Div; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +pub(super) type RoundKeys2 = [__m256i; ROUNDS]; + +type SimdBlocks = Array<__m256i, Quot>; + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn broadcast_keys(keys: &RoundKeys) -> RoundKeys2 { + keys.map(|key| _mm256_broadcastsi128_si256(key)) +} + +#[inline] +#[target_feature(enable = "vaes")] +pub(crate) unsafe fn encrypt_par( + keys: &RoundKeys2, + mut blocks: InOut<'_, '_, Array>, +) where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { + assert!(matches!(RK, 11 | 13 | 15)); + assert!(ParBlocks::USIZE % 2 == 0); + } + + let mut blocks2 = load(blocks.get_in()); + + for block2 in &mut blocks2 { + *block2 = _mm256_xor_si256(*block2, keys[0]); + } + for key in &keys[1..RK - 1] { + for block2 in &mut blocks2 { + *block2 = _mm256_aesenc_epi128(*block2, *key); + } + } + for block2 in &mut blocks2 { + *block2 = _mm256_aesenclast_epi128(*block2, keys[RK - 1]); + } + + store(blocks.get_out(), blocks2); +} + +#[inline] +#[target_feature(enable = "vaes")] +pub(crate) unsafe fn decrypt_par( + keys: &RoundKeys2, + mut blocks: InOut<'_, '_, Array>, +) where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { + assert!(matches!(RK, 11 | 13 | 15)); + assert!(ParBlocks::USIZE % 2 == 0); + } + + let mut blocks2 = load(blocks.get_in()); + + for block2 in &mut blocks2 { + *block2 = _mm256_xor_si256(*block2, keys[0]); + } + for key in &keys[1..RK - 1] { + for block2 in &mut blocks2 { + *block2 = _mm256_aesdec_epi128(*block2, *key); + } + } + for block2 in &mut blocks2 { + *block2 = _mm256_aesdeclast_epi128(*block2, keys[RK - 1]); + } + + store(blocks.get_out(), blocks2); +} + +#[inline] +#[target_feature(enable = "avx")] +fn load(blocks: &Array) -> SimdBlocks +where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { assert!(ParBlocks::USIZE % 2 == 0) } + + let in_ptr: *const __m256i = blocks.as_ptr().cast(); + Array::from_fn(|i| unsafe { _mm256_loadu_si256(in_ptr.add(i)) }) +} + +#[inline] +#[target_feature(enable = "avx")] +fn store(dst: &mut Array, blocks: SimdBlocks) +where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { assert!(ParBlocks::USIZE % 2 == 0) } + + let out_ptr: *mut __m256i = dst.as_mut_ptr().cast(); + for (i, block) in blocks.into_iter().enumerate() { + unsafe { _mm256_storeu_si256(out_ptr.add(i), block) } + } +} diff --git a/aes/src/backends/x86_vaes512.rs b/aes/src/backends/x86_vaes512.rs new file mode 100644 index 000000000..f443e03d0 --- /dev/null +++ b/aes/src/backends/x86_vaes512.rs @@ -0,0 +1,91 @@ +use super::x86_aes::RoundKeys; +use cipher::{ + Block, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherEncBackend, + BlockCipherEncClosure, BlockSizeUser, ParBlocks, ParBlocksSizeUser, + consts::{U16, U64}, + inout::InOut, +}; + +mod encdec; + +pub(crate) type Aes128<'a> = Aes<'a, 11>; +pub(crate) type Aes192<'a> = Aes<'a, 13>; +pub(crate) type Aes256<'a> = Aes<'a, 15>; + +#[derive(Clone, Copy)] +pub(crate) struct Aes<'a, const RK: usize> { + rk: &'a RoundKeys, + rk2: encdec::RoundKeys4, +} + +impl<'a, const RK: usize> Aes<'a, RK> { + #[inline] + #[target_feature(enable = "avx512f,vaes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn encrypt( + rk: &'a RoundKeys, + f: impl BlockCipherEncClosure, + ) { + let rk2 = unsafe { encdec::broadcast_keys(rk) }; + let backend = Self { rk, rk2 }; + f.call(&backend) + } + + #[inline] + #[target_feature(enable = "avx512f,vaes")] + // TODO(MSRV-1.86): remove `unsafe` + pub(crate) unsafe fn decrypt( + rk: &'a RoundKeys, + f: impl BlockCipherDecClosure, + ) { + let rk2 = unsafe { encdec::broadcast_keys(rk) }; + let backend = Self { rk, rk2 }; + f.call(&backend) + } +} + +impl BlockSizeUser for Aes<'_, RK> { + type BlockSize = U16; +} + +// Block sizes are chosen based on AVX-512's 32 ZMM registers. +// +// Round keys use 11, 13, 15 registers for AES-128/192/256 respectively. +// It results in 21, 19, 17 registers available for blocks with each register containing 4 blocks. +// We use the closest power-of-two value. +// TODO: bench bigger cipher-specific values +impl ParBlocksSizeUser for Aes<'_, RK> { + type ParBlocksSize = U64; +} + +impl BlockCipherEncBackend for Aes<'_, RK> { + #[inline(always)] + fn encrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "vaavx512f,vaeses")]` + unsafe { super::x86_aes::encrypt(&self.rk, block) }; + } + + #[inline(always)] + fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with + // `#[target_feature(enable = "avx512f,vaes")]` + unsafe { encdec::encrypt_par(&self.rk2, blocks) }; + } +} + +impl BlockCipherDecBackend for Aes<'_, RK> { + #[inline(always)] + fn decrypt_block(&self, block: InOut<'_, '_, Block>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "avx512f,vaes")]` + unsafe { super::x86_aes::decrypt(&self.rk, block) }; + } + + #[inline(always)] + fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { + // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with + // `#[target_feature(enable = "avx512f,vaes")]` + unsafe { encdec::decrypt_par(&self.rk2, blocks) }; + } +} diff --git a/aes/src/backends/x86_vaes512/encdec.rs b/aes/src/backends/x86_vaes512/encdec.rs new file mode 100644 index 000000000..696286bb5 --- /dev/null +++ b/aes/src/backends/x86_vaes512/encdec.rs @@ -0,0 +1,109 @@ +use super::RoundKeys; +use crate::Block; +use cipher::{Array, array::ArraySize, consts::U4, inout::InOut, typenum::Quot}; +use core::ops::Div; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +pub(super) type RoundKeys4 = [__m512i; ROUNDS]; + +type SimdBlocks = Array<__m512i, Quot>; + +#[inline] +#[target_feature(enable = "avx512f")] +pub(crate) unsafe fn broadcast_keys(keys: &RoundKeys) -> RoundKeys4 { + keys.map(|key| _mm512_broadcast_i32x4(key)) +} + +#[inline] +#[target_feature(enable = "avx512f,vaes")] +pub(crate) unsafe fn encrypt_par( + keys: &RoundKeys4, + mut blocks: InOut<'_, '_, Array>, +) where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { + assert!(matches!(RK, 11 | 13 | 15)); + assert!(ParBlocks::USIZE % 4 == 0); + } + + let mut blocks4 = load(blocks.get_in()); + + for block4 in &mut blocks4 { + *block4 = _mm512_xor_si512(*block4, keys[0]); + } + for key in &keys[1..RK - 1] { + for block4 in &mut blocks4 { + *block4 = _mm512_aesenc_epi128(*block4, *key); + } + } + for block4 in &mut blocks4 { + *block4 = _mm512_aesenclast_epi128(*block4, keys[RK - 1]); + } + + store(blocks.get_out(), blocks4); +} + +#[inline] +#[target_feature(enable = "avx512f,vaes")] +pub(crate) unsafe fn decrypt_par( + keys: &RoundKeys4, + mut blocks: InOut<'_, '_, Array>, +) where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { + assert!(matches!(RK, 11 | 13 | 15)); + assert!(ParBlocks::USIZE % 4 == 0); + } + + let mut blocks4 = load(blocks.get_in()); + + for block4 in &mut blocks4 { + *block4 = _mm512_xor_si512(*block4, keys[0]); + } + for key in &keys[1..RK - 1] { + for block4 in &mut blocks4 { + *block4 = _mm512_aesdec_epi128(*block4, *key); + } + } + for block4 in &mut blocks4 { + *block4 = _mm512_aesdeclast_epi128(*block4, keys[RK - 1]); + } + + store(blocks.get_out(), blocks4); +} + +#[inline] +#[target_feature(enable = "avx512f")] +fn load(blocks: &Array) -> SimdBlocks +where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { assert!(ParBlocks::USIZE % 4 == 0) } + + let in_ptr: *const __m512i = blocks.as_ptr().cast(); + Array::from_fn(|i| unsafe { _mm512_loadu_si512(in_ptr.add(i)) }) +} + +#[inline] +#[target_feature(enable = "avx512f")] +fn store(dst: &mut Array, blocks: SimdBlocks) +where + ParBlocks: ArraySize + Div, + Quot: ArraySize, +{ + const { assert!(ParBlocks::USIZE % 4 == 0) } + + let out_ptr: *mut __m512i = dst.as_mut_ptr().cast(); + for (i, block) in blocks.into_iter().enumerate() { + unsafe { _mm512_storeu_si512(out_ptr.add(i), block) } + } +} diff --git a/aes/src/hazmat.rs b/aes/src/hazmat.rs index f794aa5e4..6a3eeaa52 100644 --- a/aes/src/hazmat.rs +++ b/aes/src/hazmat.rs @@ -2,8 +2,8 @@ //! //! # ☢️️ WARNING: HAZARDOUS API ☢️ //! -//! This module contains an extremely low-level cryptographic primitive -//! which is likewise extremely difficult to use correctly. +//! This module contains an extremely low-level cryptographic primitives +//! which are likewise extremely difficult to use correctly. //! //! There are very few valid uses cases for this API. It's intended to be used //! for implementing well-reviewed higher-level constructions. @@ -11,27 +11,12 @@ //! We do NOT recommend using it to implement any algorithm which has not //! received extensive peer review by cryptographers. -use crate::soft::fixslice::hazmat as soft; +use crate::backends::soft::hazmat as soft; pub use crate::Block; /// Eight 128-bit AES blocks pub type Block8 = cipher::array::Array; -#[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] -use crate::armv8::hazmat as intrinsics; - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - not(aes_backend = "soft") -))] -use crate::x86::ni::hazmat as intrinsics; - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"), - not(aes_backend = "soft") -))] -cpufeatures::new!(aes_intrinsics, "aes"); - /// Execute the provided body if CPU intrinsics are available. // TODO(tarcieri): more `cfg-if`-like macro with an else branch? macro_rules! if_intrinsics_available { @@ -40,7 +25,12 @@ macro_rules! if_intrinsics_available { any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"), not(aes_backend = "soft") ))] - if aes_intrinsics::get() { + if crate::features_aes::get() { + #[cfg(target_arch = "aarch64")] + use crate::backends::aarch64_aes::hazmat as intrinsics; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + use crate::backends::x86_aes::hazmat as intrinsics; + unsafe { $body } return; } diff --git a/aes/src/lib.rs b/aes/src/lib.rs index 46075e788..0f1889456 100644 --- a/aes/src/lib.rs +++ b/aes/src/lib.rs @@ -125,32 +125,417 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![warn(missing_docs, rust_2018_idioms)] +pub use cipher; + #[cfg(feature = "hazmat")] pub mod hazmat; -#[macro_use] -mod macros; -mod soft; - -cpubits::cfg_if! { - if #[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] { - mod armv8; - mod autodetect; - pub use autodetect::*; - } else if #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - not(aes_backend = "soft") - ))] { - mod x86; - mod autodetect; - pub use autodetect::*; +mod backends; + +use cipher::{ + AlgorithmName, BlockCipherDecClosure, BlockCipherDecrypt, BlockCipherEncClosure, + BlockCipherEncrypt, BlockSizeUser, Key, KeyInit, KeySizeUser, + array::Array, + consts::{U16, U24, U32}, +}; +use core::fmt; +use cpubits::cfg_if; + +/// 128-bit AES block +pub type Block = Array; + +// Define token used for target feature detection +cfg_if! { + if #[cfg(aes_backend = "soft")] { + type Token = (); + } else if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + cpufeatures::new!(features_aes, "aes"); + #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] + cpufeatures::new!(features_vaes256, "vaes"); + #[cfg(aes_backend = "avx512")] + cpufeatures::new!(features_vaes512, "avx512f", "vaes"); + + #[derive(Clone, Copy)] + struct Token { + aes: features_aes::InitToken, + #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] + vaes256: features_vaes256::InitToken, + #[cfg(aes_backend = "avx512")] + vaes512: features_vaes512::InitToken, + } + + impl Default for Token { + fn default() -> Self { + Token { + aes: features_aes::InitToken::init(), + #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] + vaes256: features_vaes256::InitToken::init(), + #[cfg(aes_backend = "avx512")] + vaes512: features_vaes512::InitToken::init(), + } + } + } + + } else if #[cfg(target_arch = "aarch64")] { + cpufeatures::new!(features_aes, "aes"); + + #[derive(Clone, Copy)] + struct Token { + aes: features_aes::InitToken, + } + + impl Default for Token { + fn default() -> Self { + Token { + aes: features_aes::InitToken::init(), + } + } + } } else { - pub use soft::*; + type Token = (); } } -pub use cipher; -use cipher::{array::Array, consts::U16}; +macro_rules! impl_key_init { + ($name:ident, $soft_name:ident, $key_size:ty, $inner:path) => { + impl KeySizeUser for $name { + type KeySize = $key_size; + } -/// 128-bit AES block -pub type Block = Array; + impl KeyInit for $name { + #[inline] + fn new(key: &Key) -> Self { + type Inner = $inner; + let token = Token::default(); + let key = &key.0; + + #[cfg(not(aes_backend = "soft"))] + cfg_if! { + if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + if token.aes.get() { + // SAFETY: we confirmed that the required target features are available + let aes = unsafe { backends::x86_aes::$name::new(key) }; + let inner = Inner { aes }; + return Self { inner, token }; + } + } else if #[cfg(target_arch = "aarch64")] { + if token.aes.get() { + // SAFETY: we confirmed that the required target features are available + let aes = unsafe { backends::aarch64_aes::$name::new(key) }; + let inner = Inner { aes }; + return Self { inner, token }; + } + } + } + + let soft = backends::soft::$soft_name::new(key); + let inner = Inner { soft }; + Self { inner, token } + } + } + }; +} + +macro_rules! impl_encrypt { + ($ty_name:ident, $name:ident) => { + impl BlockCipherEncrypt for $ty_name { + #[inline] + fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { + #[cfg(not(aes_backend = "soft"))] + cfg_if! { + if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + #[cfg(aes_backend = "avx512")] + if self.token.vaes512.get() { + // SAFETY: we access correct union variant + let enc_rk = unsafe { &self.inner.aes.enc_rk }; + // SAFETY: we confirmed that the required target features are available + unsafe { backends::x86_vaes512::$name::encrypt(enc_rk, f) }; + return; + } + + #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] + if self.token.vaes256.get() { + // SAFETY: we access correct union variant + let enc_rk = unsafe { &self.inner.aes.enc_rk }; + // SAFETY: we confirmed that the required target features are available + unsafe { backends::x86_vaes256::$name::encrypt(enc_rk, f) }; + return; + } + + if self.token.aes.get() { + // SAFETY: we access correct union variant + let aes = unsafe { &self.inner.aes }; + // SAFETY: we confirmed that the required target features are available + unsafe { aes.encrypt(f) }; + return; + } + } else if #[cfg(target_arch = "aarch64")] { + if self.token.aes.get() { + // SAFETY: we access correct union variant + let aes = unsafe { &self.inner.aes }; + // SAFETY: we confirmed that the required target features are available + unsafe { aes.encrypt(f) }; + return; + } + } + } + + // SAFETY: we access correct union variant + let backend = unsafe { &self.inner.soft }; + f.call(backend); + } + } + }; +} + +macro_rules! impl_decrypt { + ($name:ident, $alg_name:ident) => { + impl BlockCipherDecrypt for $name { + #[inline] + fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { + #[cfg(not(aes_backend = "soft"))] + cfg_if! { + if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + #[cfg(aes_backend = "avx512")] + if self.token.vaes512.get() { + // SAFETY: we access correct union variant + let dec_rk = unsafe { &self.inner.aes.dec_rk }; + // SAFETY: we confirmed that the required target features are available + unsafe { backends::x86_vaes512::$alg_name::decrypt(dec_rk, f) }; + return; + } + + #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] + if self.token.vaes256.get() { + // SAFETY: we access correct union variant + let dec_rk = unsafe { &self.inner.aes.dec_rk }; + // SAFETY: we confirmed that the required target features are available + unsafe { backends::x86_vaes256::$alg_name::decrypt(dec_rk, f) }; + return; + } + + if self.token.aes.get() { + // SAFETY: we access correct union variant + let backend = unsafe { &self.inner.aes }; + // SAFETY: we confirmed that the required target features are available + unsafe { backend.decrypt(f) }; + return; + } + } else if #[cfg(target_arch = "aarch64")] { + if self.token.aes.get() { + // SAFETY: we access correct union variant + let backend = unsafe { &self.inner.aes }; + // SAFETY: we confirmed that the required target features are available + unsafe { backend.decrypt(f) }; + return; + } + } + } + + // SAFETY: we access correct union variant + let backend = unsafe { &self.inner.soft }; + f.call(backend); + } + } + }; +} + +macro_rules! impl_from_enc { + ($name:ident, $name_enc:ident, $inner:path, $into_fn:ident) => { + impl From<&$name_enc> for $name { + #[inline] + fn from(enc: &$name_enc) -> $name { + type Inner = $inner; + + let token = enc.token; + + #[cfg(not(aes_backend = "soft"))] + cfg_if! { + if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + if token.aes.get() { + // SAFETY: we access correct union variant + let aes_enc = unsafe { &enc.inner.aes }; + // SAFETY: we confirmed that the required target features are available + let aes = unsafe { aes_enc.$into_fn() }; + let inner = Inner { aes }; + return Self { inner, token }; + } + } else if #[cfg(target_arch = "aarch64")] { + if token.aes.get() { + // SAFETY: we access correct union variant + let aes_enc = unsafe { &enc.inner.aes }; + // SAFETY: we confirmed that the required target features are available + let aes = unsafe { aes_enc.$into_fn() }; + let inner = Inner { aes }; + return Self { inner, token }; + } + } + } + + // SAFETY: we access correct union variant + let soft = unsafe { enc.inner.soft }; + let inner = Inner { soft }; + Self { inner, token } + } + } + + impl From<$name_enc> for $name { + #[inline] + fn from(enc: $name_enc) -> $name { + Self::from(&enc) + } + } + }; +} + +macro_rules! common_impls { + ($name:ident) => { + impl Clone for $name { + #[inline] + fn clone(&self) -> Self { + unsafe { core::ptr::read(self) } + } + } + + impl BlockSizeUser for $name { + type BlockSize = U16; + } + + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.write_str(concat!(stringify!($name), " { .. }")) + } + } + + impl AlgorithmName for $name { + fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(stringify!($name)) + } + } + + impl Drop for $name { + #[inline] + fn drop(&mut self) { + #[cfg(feature = "zeroize")] + unsafe { + zeroize::zeroize_flat_type(self); + } + } + } + + #[cfg(feature = "zeroize")] + impl zeroize::ZeroizeOnDrop for $name {} + }; +} + +macro_rules! define_aes_impl { + ( + name = $name:ident, + name_enc = $name_enc:ident, + name_dec = $name_dec:ident, + module = $module:tt, + key_size = $key_size:ident, + doc = $doc:expr, + ) => { + mod $module { + use crate::backends; + + pub(super) union Inner { + #[cfg(all( + any(target_arch = "x86_64", target_arch = "x86"), + not(aes_backend = "soft"), + ))] + pub(super) aes: backends::x86_aes::$name, + #[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] + pub(super) aes: backends::aarch64_aes::$name, + pub(super) soft: backends::soft::$name, + } + + pub(super) union InnerEnc { + #[cfg(all( + any(target_arch = "x86_64", target_arch = "x86"), + not(aes_backend = "soft"), + ))] + pub(super) aes: backends::x86_aes::$name_enc, + #[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] + pub(super) aes: backends::aarch64_aes::$name_enc, + pub(super) soft: backends::soft::$name, + } + + pub(super) union InnerDec { + #[cfg(all( + any(target_arch = "x86_64", target_arch = "x86"), + not(aes_backend = "soft"), + ))] + pub(super) aes: backends::x86_aes::$name_dec, + #[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] + pub(super) aes: backends::aarch64_aes::$name_dec, + pub(super) soft: backends::soft::$name, + } + } + + #[doc=$doc] + #[doc = "block cipher"] + pub struct $name { + inner: $module::Inner, + #[allow(dead_code, reason = "this field is not used on software-only targets")] + token: Token, + } + + common_impls!($name); + impl_key_init!($name, $name, $key_size, $module::Inner); + impl_encrypt!($name, $name); + impl_decrypt!($name, $name); + impl_from_enc!($name, $name_enc, $module::Inner, as_encdec); + + #[doc=$doc] + #[doc = "block cipher (encrypt-only)"] + pub struct $name_enc { + inner: $module::InnerEnc, + #[allow(dead_code, reason = "this field is not used on software-only targets")] + token: Token, + } + + common_impls!($name_enc); + impl_key_init!($name_enc, $name, $key_size, $module::InnerEnc); + impl_encrypt!($name_enc, $name); + + #[doc=$doc] + #[doc = "block cipher (decrypt-only)"] + pub struct $name_dec { + inner: $module::InnerDec, + #[allow(dead_code, reason = "this field is not used on software-only targets")] + token: Token, + } + + common_impls!($name_dec); + impl_key_init!($name_dec, $name, $key_size, $module::InnerDec); + impl_decrypt!($name_dec, $name); + impl_from_enc!($name_dec, $name_enc, $module::InnerDec, as_dec); + }; +} + +define_aes_impl!( + name = Aes128, + name_enc = Aes128Enc, + name_dec = Aes128Dec, + module = aes128, + key_size = U16, + doc = "AES-128", +); +define_aes_impl!( + name = Aes192, + name_enc = Aes192Enc, + name_dec = Aes192Dec, + module = aes192, + key_size = U24, + doc = "AES-192", +); +define_aes_impl!( + name = Aes256, + name_enc = Aes256Enc, + name_dec = Aes256Dec, + module = aes256, + key_size = U32, + doc = "AES-256", +); diff --git a/aes/src/macros.rs b/aes/src/macros.rs deleted file mode 100644 index e688c00d9..000000000 --- a/aes/src/macros.rs +++ /dev/null @@ -1,105 +0,0 @@ -// This macro is not used by the soft backend, to simplify the crate code we allow this macro -// to be unused to prevent warnings e.g. when `force-soft` is enabled/ -#[allow(unused_macros)] -macro_rules! impl_backends { - ( - enc_name = $enc_name:ident, - dec_name = $dec_name:ident, - key_size = $key_size:ty, - keys_ty = $keys_ty:ty, - par_size = $par_size:ty, - expand_keys = $expand_keys:expr, - inv_keys = $inv_keys:expr, - encrypt = $encrypt:expr, - encrypt_par = $encrypt_par:expr, - decrypt = $decrypt:expr, - decrypt_par = $decrypt_par:expr, -) => { - #[derive(Clone)] - pub(crate) struct $enc_name { - keys: $keys_ty, - } - - impl cipher::BlockSizeUser for $enc_name { - type BlockSize = cipher::consts::U16; - } - - impl cipher::ParBlocksSizeUser for $enc_name { - type ParBlocksSize = $par_size; - } - - impl cipher::KeySizeUser for $enc_name { - type KeySize = $key_size; - } - - impl cipher::KeyInit for $enc_name { - #[inline] - fn new(key: &cipher::Key) -> Self { - let keys = unsafe { $expand_keys(key.as_ref()) }; - Self { keys } - } - } - - impl cipher::BlockCipherEncBackend for $enc_name { - #[inline(always)] - fn encrypt_block(&self, block: cipher::inout::InOut<'_, '_, cipher::Block>) { - unsafe { $encrypt(&self.keys, block) } - } - - #[inline(always)] - fn encrypt_par_blocks( - &self, - blocks: cipher::inout::InOut<'_, '_, cipher::ParBlocks>, - ) { - unsafe { $encrypt_par(&self.keys, blocks) } - } - } - - #[derive(Clone)] - pub(crate) struct $dec_name { - keys: $keys_ty, - } - - impl cipher::BlockSizeUser for $dec_name { - type BlockSize = cipher::consts::U16; - } - - impl cipher::ParBlocksSizeUser for $dec_name { - type ParBlocksSize = $par_size; - } - - impl cipher::KeySizeUser for $dec_name { - type KeySize = $key_size; - } - - impl cipher::KeyInit for $dec_name { - #[inline] - fn new(key: &cipher::Key) -> Self { - From::from($enc_name::new(key)) - } - } - - impl From<$enc_name> for $dec_name { - #[inline] - fn from(enc: $enc_name) -> $dec_name { - let keys = unsafe { $inv_keys(&enc.keys) }; - Self { keys } - } - } - - impl cipher::BlockCipherDecBackend for $dec_name { - #[inline(always)] - fn decrypt_block(&self, block: cipher::inout::InOut<'_, '_, cipher::Block>) { - unsafe { $decrypt(&self.keys, block) } - } - - #[inline(always)] - fn decrypt_par_blocks( - &self, - blocks: cipher::inout::InOut<'_, '_, cipher::ParBlocks>, - ) { - unsafe { $decrypt_par(&self.keys, blocks) } - } - } - }; -} diff --git a/aes/src/ni.rs b/aes/src/ni.rs deleted file mode 100644 index f9fa5011d..000000000 --- a/aes/src/ni.rs +++ /dev/null @@ -1,339 +0,0 @@ -//! AES block ciphers implementation using AES-NI instruction set. -//! -//! Ciphers functionality is accessed using `BlockCipher` trait from the -//! [`cipher`](https://docs.rs/cipher) crate. -//! -//! # Vulnerability -//! Lazy FP state restory vulnerability can allow local process to leak content -//! of the FPU register, in which round keys are stored. This vulnerability -//! can be mitigated at the operating system level by installing relevant -//! patches. (i.e. keep your OS updated!) More info: -//! - [Intel advisory](https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00145.html) -//! - [Wikipedia](https://en.wikipedia.org/wiki/Lazy_FP_state_restore) -//! -//! # Related documents -//! - [Intel AES-NI whitepaper](https://software.intel.com/sites/default/files/article/165683/aes-wp-2012-09-22-v01.pdf) -//! - [Use of the AES Instruction Set](https://www.cosic.esat.kuleuven.be/ecrypt/AESday/slides/Use_of_the_AES_Instruction_Set.pdf) - -mod encdec; -mod expand; -#[cfg(test)] -mod test_expand; - -#[cfg(feature = "hazmat")] -pub(crate) mod hazmat; - -#[cfg(target_arch = "x86")] -use core::arch::x86 as arch; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64 as arch; - -use cipher::{ - AlgorithmName, BlockCipherDecClosure, BlockCipherDecrypt, BlockCipherEncClosure, - BlockCipherEncrypt, BlockSizeUser, Key, KeyInit, KeySizeUser, - consts::{self, U16, U24, U32}, -}; -use core::fmt; - -impl_backends!( - enc_name = Aes128BackEnc, - dec_name = Aes128BackDec, - key_size = consts::U16, - keys_ty = expand::Aes128RoundKeys, - par_size = consts::U9, - expand_keys = expand::aes128_expand_key, - inv_keys = expand::inv_keys, - encrypt = encdec::encrypt, - encrypt_par = encdec::encrypt_par, - decrypt = encdec::decrypt, - decrypt_par = encdec::decrypt_par, -); - -impl_backends!( - enc_name = Aes192BackEnc, - dec_name = Aes192BackDec, - key_size = consts::U24, - keys_ty = expand::Aes192RoundKeys, - par_size = consts::U9, - expand_keys = expand::aes192_expand_key, - inv_keys = expand::inv_keys, - encrypt = encdec::encrypt, - encrypt_par = encdec::encrypt_par, - decrypt = encdec::decrypt, - decrypt_par = encdec::decrypt_par, -); - -impl_backends!( - enc_name = Aes256BackEnc, - dec_name = Aes256BackDec, - key_size = consts::U32, - keys_ty = expand::Aes256RoundKeys, - par_size = consts::U9, - expand_keys = expand::aes256_expand_key, - inv_keys = expand::inv_keys, - encrypt = encdec::encrypt, - encrypt_par = encdec::encrypt_par, - decrypt = encdec::decrypt, - decrypt_par = encdec::decrypt_par, -); - -macro_rules! define_aes_impl { - ( - $name:tt, - $name_enc:ident, - $name_dec:ident, - $name_back_enc:ident, - $name_back_dec:ident, - $key_size:ty, - $doc:expr $(,)? - ) => { - #[doc=$doc] - #[doc = "block cipher"] - #[derive(Clone)] - pub struct $name { - encrypt: $name_enc, - decrypt: $name_dec, - } - - impl $name { - #[inline(always)] - pub(crate) fn get_enc_backend(&self) -> &$name_back_enc { - self.encrypt.get_enc_backend() - } - - #[inline(always)] - pub(crate) fn get_dec_backend(&self) -> &$name_back_dec { - self.decrypt.get_dec_backend() - } - } - - impl KeySizeUser for $name { - type KeySize = $key_size; - } - - impl KeyInit for $name { - #[inline] - fn new(key: &Key) -> Self { - let encrypt = $name_enc::new(key); - let decrypt = $name_dec::from(&encrypt); - Self { encrypt, decrypt } - } - } - - impl From<$name_enc> for $name { - #[inline] - fn from(encrypt: $name_enc) -> $name { - let decrypt = (&encrypt).into(); - Self { encrypt, decrypt } - } - } - - impl From<&$name_enc> for $name { - #[inline] - fn from(encrypt: &$name_enc) -> $name { - let decrypt = encrypt.into(); - let encrypt = encrypt.clone(); - Self { encrypt, decrypt } - } - } - - impl BlockSizeUser for $name { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - self.encrypt.encrypt_with_backend(f) - } - } - - impl BlockCipherDecrypt for $name { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - self.decrypt.decrypt_with_backend(f) - } - } - - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name), " { .. }")) - } - } - - impl AlgorithmName for $name { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name)) - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name {} - - #[doc=$doc] - #[doc = "block cipher (encrypt-only)"] - #[derive(Clone)] - pub struct $name_enc { - backend: $name_back_enc, - } - - impl $name_enc { - #[inline(always)] - pub(crate) fn get_enc_backend(&self) -> &$name_back_enc { - &self.backend - } - } - - impl KeySizeUser for $name_enc { - type KeySize = $key_size; - } - - impl KeyInit for $name_enc { - #[inline] - fn new(key: &Key) -> Self { - Self { - backend: $name_back_enc::new(key), - } - } - } - - impl BlockSizeUser for $name_enc { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name_enc { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - f.call(&self.backend) - } - } - - impl fmt::Debug for $name_enc { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_enc), " { .. }")) - } - } - - impl AlgorithmName for $name_enc { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_enc)) - } - } - - impl Drop for $name_enc { - #[inline] - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(&mut self.backend) - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_enc {} - - #[doc=$doc] - #[doc = "block cipher (decrypt-only)"] - #[derive(Clone)] - pub struct $name_dec { - backend: $name_back_dec, - } - - impl $name_dec { - #[inline(always)] - pub(crate) fn get_dec_backend(&self) -> &$name_back_dec { - &self.backend - } - } - - impl KeySizeUser for $name_dec { - type KeySize = $key_size; - } - - impl KeyInit for $name_dec { - #[inline] - fn new(key: &Key) -> Self { - $name_enc::new(key).into() - } - } - - impl From<$name_enc> for $name_dec { - #[inline] - fn from(enc: $name_enc) -> $name_dec { - Self::from(&enc) - } - } - - impl From<&$name_enc> for $name_dec { - #[inline] - fn from(enc: &$name_enc) -> $name_dec { - Self { - backend: enc.backend.clone().into(), - } - } - } - - impl BlockSizeUser for $name_dec { - type BlockSize = U16; - } - - impl BlockCipherDecrypt for $name_dec { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - f.call(self.get_dec_backend()); - } - } - - impl fmt::Debug for $name_dec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_dec), " { .. }")) - } - } - - impl AlgorithmName for $name_dec { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_dec)) - } - } - - impl Drop for $name_dec { - #[inline] - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(&mut self.backend) - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_dec {} - }; -} - -define_aes_impl!( - Aes128, - Aes128Enc, - Aes128Dec, - Aes128BackEnc, - Aes128BackDec, - U16, - "AES-128", -); - -define_aes_impl!( - Aes192, - Aes192Enc, - Aes192Dec, - Aes192BackEnc, - Aes192BackDec, - U24, - "AES-192", -); - -define_aes_impl!( - Aes256, - Aes256Enc, - Aes256Dec, - Aes256BackEnc, - Aes256BackDec, - U32, - "AES-256", -); diff --git a/aes/src/ni/hazmat.rs b/aes/src/ni/hazmat.rs deleted file mode 100644 index 0e9a9cd2b..000000000 --- a/aes/src/ni/hazmat.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! Low-level "hazmat" AES functions: AES-NI support. -//! -//! Note: this isn't actually used in the `Aes128`/`Aes192`/`Aes256` -//! implementations in this crate, but instead provides raw AES-NI accelerated -//! access to the AES round function gated under the `hazmat` crate feature. -#![allow(unsafe_op_in_unsafe_fn)] - -use super::arch::*; -use crate::hazmat::{Block, Block8}; -use cipher::array::{Array, ArraySize}; - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn load(blocks: *const Array) -> Array<__m128i, N> { - let p = blocks.cast::<__m128i>(); - let mut res: Array<__m128i, N> = core::mem::zeroed(); - for i in 0..N::USIZE { - res[i] = _mm_loadu_si128(p.add(i)); - } - res -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn store(blocks: *mut Array, b: Array<__m128i, N>) { - let p = blocks.cast::<__m128i>(); - for i in 0..N::USIZE { - _mm_storeu_si128(p.add(i), b[i]); - } -} - -/// AES cipher (encrypt) round function. -#[target_feature(enable = "aes")] -pub(crate) unsafe fn cipher_round(block: &mut Block, round_key: &Block) { - // Safety: `loadu` and `storeu` support unaligned access - let b = _mm_loadu_si128(block.as_ptr() as *const __m128i); - let k = _mm_loadu_si128(round_key.as_ptr() as *const __m128i); - let out = _mm_aesenc_si128(b, k); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, out); -} - -/// AES cipher (encrypt) round function: parallel version. -#[target_feature(enable = "aes")] -pub(crate) unsafe fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { - let xmm_keys = load(round_keys); - let mut xmm_blocks = load(blocks); - - for i in 0..8 { - xmm_blocks[i] = _mm_aesenc_si128(xmm_blocks[i], xmm_keys[i]); - } - - store(blocks, xmm_blocks); -} - -/// AES cipher (encrypt) round function. -#[target_feature(enable = "aes")] -pub(crate) unsafe fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { - // Safety: `loadu` and `storeu` support unaligned access - let b = _mm_loadu_si128(block.as_ptr() as *const __m128i); - let k = _mm_loadu_si128(round_key.as_ptr() as *const __m128i); - let out = _mm_aesdec_si128(b, k); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, out); -} - -/// AES cipher (encrypt) round function: parallel version. -#[target_feature(enable = "aes")] -pub(crate) unsafe fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { - let xmm_keys = load(round_keys); - let mut xmm_blocks = load(blocks); - - for i in 0..8 { - xmm_blocks[i] = _mm_aesdec_si128(xmm_blocks[i], xmm_keys[i]); - } - - store(blocks, xmm_blocks); -} - -/// AES mix columns function. -#[target_feature(enable = "aes")] -pub(crate) unsafe fn mix_columns(block: &mut Block) { - // Safety: `loadu` and `storeu` support unaligned access - let mut state = _mm_loadu_si128(block.as_ptr() as *const __m128i); - - // Emulate mix columns by performing three inverse mix columns operations - state = _mm_aesimc_si128(state); - state = _mm_aesimc_si128(state); - state = _mm_aesimc_si128(state); - - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, state); -} - -/// AES inverse mix columns function. -#[target_feature(enable = "aes")] -pub(crate) unsafe fn inv_mix_columns(block: &mut Block) { - // Safety: `loadu` and `storeu` support unaligned access - let b = _mm_loadu_si128(block.as_ptr() as *const __m128i); - let out = _mm_aesimc_si128(b); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, out); -} diff --git a/aes/src/ni/test_expand.rs b/aes/src/ni/test_expand.rs deleted file mode 100644 index 973f827d3..000000000 --- a/aes/src/ni/test_expand.rs +++ /dev/null @@ -1,290 +0,0 @@ -use super::{arch::*, expand::*}; -use hex_literal::hex; - -pub(crate) fn check(a: &[__m128i], b: &[[u64; 2]]) { - assert_eq!(a.len(), b.len()); - for (v1, v2) in a.iter().zip(b) { - let t1: [u64; 2] = unsafe { core::mem::transmute(*v1) }; - let t2 = [v2[0].to_be(), v2[1].to_be()]; - assert_eq!(t1, t2); - } -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes128_expand_key_test() { - let keys = [0x00; 16]; - check( - &unsafe { aes128_expand_key(&keys) }, - &[ - [0x0000000000000000, 0x0000000000000000], - [0x6263636362636363, 0x6263636362636363], - [0x9b9898c9f9fbfbaa, 0x9b9898c9f9fbfbaa], - [0x90973450696ccffa, 0xf2f457330b0fac99], - [0xee06da7b876a1581, 0x759e42b27e91ee2b], - [0x7f2e2b88f8443e09, 0x8dda7cbbf34b9290], - [0xec614b851425758c, 0x99ff09376ab49ba7], - [0x217517873550620b, 0xacaf6b3cc61bf09b], - [0x0ef903333ba96138, 0x97060a04511dfa9f], - [0xb1d4d8e28a7db9da, 0x1d7bb3de4c664941], - [0xb4ef5bcb3e92e211, 0x23e951cf6f8f188e], - ], - ); - - let keys = [0xff; 16]; - check( - &unsafe { aes128_expand_key(&keys) }, - &[ - [0xffffffffffffffff, 0xffffffffffffffff], - [0xe8e9e9e917161616, 0xe8e9e9e917161616], - [0xadaeae19bab8b80f, 0x525151e6454747f0], - [0x090e2277b3b69a78, 0xe1e7cb9ea4a08c6e], - [0xe16abd3e52dc2746, 0xb33becd8179b60b6], - [0xe5baf3ceb766d488, 0x045d385013c658e6], - [0x71d07db3c6b6a93b, 0xc2eb916bd12dc98d], - [0xe90d208d2fbb89b6, 0xed5018dd3c7dd150], - [0x96337366b988fad0, 0x54d8e20d68a5335d], - [0x8bf03f233278c5f3, 0x66a027fe0e0514a3], - [0xd60a3588e472f07b, 0x82d2d7858cd7c326], - ], - ); - - let keys = hex!("000102030405060708090a0b0c0d0e0f"); - check( - &unsafe { aes128_expand_key(&keys) }, - &[ - [0x0001020304050607, 0x08090a0b0c0d0e0f], - [0xd6aa74fdd2af72fa, 0xdaa678f1d6ab76fe], - [0xb692cf0b643dbdf1, 0xbe9bc5006830b3fe], - [0xb6ff744ed2c2c9bf, 0x6c590cbf0469bf41], - [0x47f7f7bc95353e03, 0xf96c32bcfd058dfd], - [0x3caaa3e8a99f9deb, 0x50f3af57adf622aa], - [0x5e390f7df7a69296, 0xa7553dc10aa31f6b], - [0x14f9701ae35fe28c, 0x440adf4d4ea9c026], - [0x47438735a41c65b9, 0xe016baf4aebf7ad2], - [0x549932d1f0855768, 0x1093ed9cbe2c974e], - [0x13111d7fe3944a17, 0xf307a78b4d2b30c5], - ], - ); - - let keys = hex!("6920e299a5202a6d656e636869746f2a"); - check( - &unsafe { aes128_expand_key(&keys) }, - &[ - [0x6920e299a5202a6d, 0x656e636869746f2a], - [0xfa8807605fa82d0d, 0x3ac64e6553b2214f], - [0xcf75838d90ddae80, 0xaa1be0e5f9a9c1aa], - [0x180d2f1488d08194, 0x22cb6171db62a0db], - [0xbaed96ad323d1739, 0x10f67648cb94d693], - [0x881b4ab2ba265d8b, 0xaad02bc36144fd50], - [0xb34f195d096944d6, 0xa3b96f15c2fd9245], - [0xa7007778ae6933ae, 0x0dd05cbbcf2dcefe], - [0xff8bccf251e2ff5c, 0x5c32a3e7931f6d19], - [0x24b7182e7555e772, 0x29674495ba78298c], - [0xae127cdadb479ba8, 0xf220df3d4858f6b1], - ], - ); - - let keys = hex!("2b7e151628aed2a6abf7158809cf4f3c"); - check( - &unsafe { aes128_expand_key(&keys) }, - &[ - [0x2b7e151628aed2a6, 0xabf7158809cf4f3c], - [0xa0fafe1788542cb1, 0x23a339392a6c7605], - [0xf2c295f27a96b943, 0x5935807a7359f67f], - [0x3d80477d4716fe3e, 0x1e237e446d7a883b], - [0xef44a541a8525b7f, 0xb671253bdb0bad00], - [0xd4d1c6f87c839d87, 0xcaf2b8bc11f915bc], - [0x6d88a37a110b3efd, 0xdbf98641ca0093fd], - [0x4e54f70e5f5fc9f3, 0x84a64fb24ea6dc4f], - [0xead27321b58dbad2, 0x312bf5607f8d292f], - [0xac7766f319fadc21, 0x28d12941575c006e], - [0xd014f9a8c9ee2589, 0xe13f0cc8b6630ca6], - ], - ); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes192_expand_key_test() { - let keys = [0x00; 24]; - check( - &unsafe { aes192_expand_key(&keys) }, - &[ - [0x0000000000000000, 0x0000000000000000], - [0x0000000000000000, 0x6263636362636363], - [0x6263636362636363, 0x6263636362636363], - [0x9b9898c9f9fbfbaa, 0x9b9898c9f9fbfbaa], - [0x9b9898c9f9fbfbaa, 0x90973450696ccffa], - [0xf2f457330b0fac99, 0x90973450696ccffa], - [0xc81d19a9a171d653, 0x53858160588a2df9], - [0xc81d19a9a171d653, 0x7bebf49bda9a22c8], - [0x891fa3a8d1958e51, 0x198897f8b8f941ab], - [0xc26896f718f2b43f, 0x91ed1797407899c6], - [0x59f00e3ee1094f95, 0x83ecbc0f9b1e0830], - [0x0af31fa74a8b8661, 0x137b885ff272c7ca], - [0x432ac886d834c0b6, 0xd2c7df11984c5970], - ], - ); - - let keys = [0xff; 24]; - check( - &unsafe { aes192_expand_key(&keys) }, - &[ - [0xffffffffffffffff, 0xffffffffffffffff], - [0xffffffffffffffff, 0xe8e9e9e917161616], - [0xe8e9e9e917161616, 0xe8e9e9e917161616], - [0xadaeae19bab8b80f, 0x525151e6454747f0], - [0xadaeae19bab8b80f, 0xc5c2d8ed7f7a60e2], - [0x2d2b3104686c76f4, 0xc5c2d8ed7f7a60e2], - [0x1712403f686820dd, 0x454311d92d2f672d], - [0xe8edbfc09797df22, 0x8f8cd3b7e7e4f36a], - [0xa2a7e2b38f88859e, 0x67653a5ef0f2e57c], - [0x2655c33bc1b13051, 0x6316d2e2ec9e577c], - [0x8bfb6d227b09885e, 0x67919b1aa620ab4b], - [0xc53679a929a82ed5, 0xa25343f7d95acba9], - [0x598e482fffaee364, 0x3a989acd1330b418], - ], - ); - - let keys = hex!("000102030405060708090a0b0c0d0e0f1011121314151617"); - check( - &unsafe { aes192_expand_key(&keys) }, - &[ - [0x0001020304050607, 0x08090a0b0c0d0e0f], - [0x1011121314151617, 0x5846f2f95c43f4fe], - [0x544afef55847f0fa, 0x4856e2e95c43f4fe], - [0x40f949b31cbabd4d, 0x48f043b810b7b342], - [0x58e151ab04a2a555, 0x7effb5416245080c], - [0x2ab54bb43a02f8f6, 0x62e3a95d66410c08], - [0xf501857297448d7e, 0xbdf1c6ca87f33e3c], - [0xe510976183519b69, 0x34157c9ea351f1e0], - [0x1ea0372a99530916, 0x7c439e77ff12051e], - [0xdd7e0e887e2fff68, 0x608fc842f9dcc154], - [0x859f5f237a8d5a3d, 0xc0c02952beefd63a], - [0xde601e7827bcdf2c, 0xa223800fd8aeda32], - [0xa4970a331a78dc09, 0xc418c271e3a41d5d], - ], - ); - - let keys = hex!("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"); - check( - &unsafe { aes192_expand_key(&keys) }, - &[ - [0x8e73b0f7da0e6452, 0xc810f32b809079e5], - [0x62f8ead2522c6b7b, 0xfe0c91f72402f5a5], - [0xec12068e6c827f6b, 0x0e7a95b95c56fec2], - [0x4db7b4bd69b54118, 0x85a74796e92538fd], - [0xe75fad44bb095386, 0x485af05721efb14f], - [0xa448f6d94d6dce24, 0xaa326360113b30e6], - [0xa25e7ed583b1cf9a, 0x27f939436a94f767], - [0xc0a69407d19da4e1, 0xec1786eb6fa64971], - [0x485f703222cb8755, 0xe26d135233f0b7b3], - [0x40beeb282f18a259, 0x6747d26b458c553e], - [0xa7e1466c9411f1df, 0x821f750aad07d753], - [0xca4005388fcc5006, 0x282d166abc3ce7b5], - [0xe98ba06f448c773c, 0x8ecc720401002202], - ], - ); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes256_expand_key_test() { - let keys = [0x00; 32]; - check( - &unsafe { aes256_expand_key(&keys) }, - &[ - [0x0000000000000000, 0x0000000000000000], - [0x0000000000000000, 0x0000000000000000], - [0x6263636362636363, 0x6263636362636363], - [0xaafbfbfbaafbfbfb, 0xaafbfbfbaafbfbfb], - [0x6f6c6ccf0d0f0fac, 0x6f6c6ccf0d0f0fac], - [0x7d8d8d6ad7767691, 0x7d8d8d6ad7767691], - [0x5354edc15e5be26d, 0x31378ea23c38810e], - [0x968a81c141fcf750, 0x3c717a3aeb070cab], - [0x9eaa8f28c0f16d45, 0xf1c6e3e7cdfe62e9], - [0x2b312bdf6acddc8f, 0x56bca6b5bdbbaa1e], - [0x6406fd52a4f79017, 0x553173f098cf1119], - [0x6dbba90b07767584, 0x51cad331ec71792f], - [0xe7b0e89c4347788b, 0x16760b7b8eb91a62], - [0x74ed0ba1739b7e25, 0x2251ad14ce20d43b], - [0x10f80a1753bf729c, 0x45c979e7cb706385], - ], - ); - - let keys = [0xff; 32]; - check( - &unsafe { aes256_expand_key(&keys) }, - &[ - [0xffffffffffffffff, 0xffffffffffffffff], - [0xffffffffffffffff, 0xffffffffffffffff], - [0xe8e9e9e917161616, 0xe8e9e9e917161616], - [0x0fb8b8b8f0474747, 0x0fb8b8b8f0474747], - [0x4a4949655d5f5f73, 0xb5b6b69aa2a0a08c], - [0x355858dcc51f1f9b, 0xcaa7a7233ae0e064], - [0xafa80ae5f2f75596, 0x4741e30ce5e14380], - [0xeca0421129bf5d8a, 0xe318faa9d9f81acd], - [0xe60ab7d014fde246, 0x53bc014ab65d42ca], - [0xa2ec6e658b5333ef, 0x684bc946b1b3d38b], - [0x9b6c8a188f91685e, 0xdc2d69146a702bde], - [0xa0bd9f782beeac97, 0x43a565d1f216b65a], - [0xfc22349173b35ccf, 0xaf9e35dbc5ee1e05], - [0x0695ed132d7b4184, 0x6ede24559cc8920f], - [0x546d424f27de1e80, 0x88402b5b4dae355e], - ], - ); - - let keys = hex!("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"); - check( - &unsafe { aes256_expand_key(&keys) }, - &[ - [0x0001020304050607, 0x08090a0b0c0d0e0f], - [0x1011121314151617, 0x18191a1b1c1d1e1f], - [0xa573c29fa176c498, 0xa97fce93a572c09c], - [0x1651a8cd0244beda, 0x1a5da4c10640bade], - [0xae87dff00ff11b68, 0xa68ed5fb03fc1567], - [0x6de1f1486fa54f92, 0x75f8eb5373b8518d], - [0xc656827fc9a79917, 0x6f294cec6cd5598b], - [0x3de23a75524775e7, 0x27bf9eb45407cf39], - [0x0bdc905fc27b0948, 0xad5245a4c1871c2f], - [0x45f5a66017b2d387, 0x300d4d33640a820a], - [0x7ccff71cbeb4fe54, 0x13e6bbf0d261a7df], - [0xf01afafee7a82979, 0xd7a5644ab3afe640], - [0x2541fe719bf50025, 0x8813bbd55a721c0a], - [0x4e5a6699a9f24fe0, 0x7e572baacdf8cdea], - [0x24fc79ccbf0979e9, 0x371ac23c6d68de36], - ], - ); - - let keys = hex!("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"); - check( - &unsafe { aes256_expand_key(&keys) }, - &[ - [0x603deb1015ca71be, 0x2b73aef0857d7781], - [0x1f352c073b6108d7, 0x2d9810a30914dff4], - [0x9ba354118e6925af, 0xa51a8b5f2067fcde], - [0xa8b09c1a93d194cd, 0xbe49846eb75d5b9a], - [0xd59aecb85bf3c917, 0xfee94248de8ebe96], - [0xb5a9328a2678a647, 0x983122292f6c79b3], - [0x812c81addadf48ba, 0x24360af2fab8b464], - [0x98c5bfc9bebd198e, 0x268c3ba709e04214], - [0x68007bacb2df3316, 0x96e939e46c518d80], - [0xc814e20476a9fb8a, 0x5025c02d59c58239], - [0xde1369676ccc5a71, 0xfa2563959674ee15], - [0x5886ca5d2e2f31d7, 0x7e0af1fa27cf73c3], - [0x749c47ab18501dda, 0xe2757e4f7401905a], - [0xcafaaae3e4d59b34, 0x9adf6acebd10190d], - [0xfe4890d1e6188d0b, 0x046df344706c631e], - ], - ); -} diff --git a/aes/src/soft.rs b/aes/src/soft.rs deleted file mode 100644 index bbee06b68..000000000 --- a/aes/src/soft.rs +++ /dev/null @@ -1,309 +0,0 @@ -//! AES block cipher constant-time implementation. -//! -//! The implementation uses a technique called [fixslicing][1], an improved -//! form of bitslicing which represents ciphers in a way which enables -//! very efficient constant-time implementations in software. -//! -//! [1]: https://eprint.iacr.org/2020/1123.pdf - -#![deny(unsafe_code)] - -pub(crate) mod fixslice; - -use crate::Block; -use cipher::{ - AlgorithmName, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherDecrypt, - BlockCipherEncBackend, BlockCipherEncClosure, BlockCipherEncrypt, BlockSizeUser, Key, KeyInit, - KeySizeUser, ParBlocksSizeUser, - consts::{U16, U24, U32}, - inout::InOut, -}; -use core::fmt; -use fixslice::{BatchBlocks, FixsliceBlocks, FixsliceKeys128, FixsliceKeys192, FixsliceKeys256}; - -macro_rules! define_aes_impl { - ( - $name:tt, - $name_enc:ident, - $name_dec:ident, - $name_back_enc:ident, - $name_back_dec:ident, - $key_size:ty, - $fixslice_keys:ty, - $fixslice_key_schedule:path, - $fixslice_decrypt:path, - $fixslice_encrypt:path, - $doc:expr $(,)? - ) => { - #[doc=$doc] - #[doc = "block cipher"] - #[derive(Clone)] - pub struct $name { - keys: $fixslice_keys, - } - - impl KeySizeUser for $name { - type KeySize = $key_size; - } - - impl KeyInit for $name { - #[inline] - fn new(key: &Key) -> Self { - Self { - keys: $fixslice_key_schedule(key.into()), - } - } - } - - impl BlockSizeUser for $name { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - f.call(&$name_back_enc(self)) - } - } - - impl BlockCipherDecrypt for $name { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - f.call(&$name_back_dec(self)) - } - } - - impl From<$name_enc> for $name { - #[inline] - fn from(enc: $name_enc) -> $name { - enc.inner - } - } - - impl From<&$name_enc> for $name { - #[inline] - fn from(enc: &$name_enc) -> $name { - enc.inner.clone() - } - } - - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name), " { .. }")) - } - } - - impl AlgorithmName for $name { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name)) - } - } - - impl Drop for $name { - #[inline] - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - zeroize::Zeroize::zeroize(&mut self.keys); - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name {} - - #[doc=$doc] - #[doc = "block cipher (encrypt-only)"] - #[derive(Clone)] - pub struct $name_enc { - inner: $name, - } - - impl KeySizeUser for $name_enc { - type KeySize = $key_size; - } - - impl KeyInit for $name_enc { - #[inline(always)] - fn new(key: &Key) -> Self { - let inner = $name::new(key); - Self { inner } - } - } - - impl BlockSizeUser for $name_enc { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name_enc { - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - f.call(&mut $name_back_enc(&self.inner)) - } - } - - impl fmt::Debug for $name_enc { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_enc), " { .. }")) - } - } - - impl AlgorithmName for $name_enc { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_enc)) - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_enc {} - - #[doc=$doc] - #[doc = "block cipher (decrypt-only)"] - #[derive(Clone)] - pub struct $name_dec { - inner: $name, - } - - impl KeySizeUser for $name_dec { - type KeySize = $key_size; - } - - impl KeyInit for $name_dec { - #[inline(always)] - fn new(key: &Key) -> Self { - let inner = $name::new(key); - Self { inner } - } - } - - impl From<$name_enc> for $name_dec { - #[inline] - fn from(enc: $name_enc) -> $name_dec { - Self { inner: enc.inner } - } - } - - impl From<&$name_enc> for $name_dec { - #[inline] - fn from(enc: &$name_enc) -> $name_dec { - Self { - inner: enc.inner.clone(), - } - } - } - - impl BlockSizeUser for $name_dec { - type BlockSize = U16; - } - - impl BlockCipherDecrypt for $name_dec { - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - f.call(&$name_back_dec(&self.inner)); - } - } - - impl fmt::Debug for $name_dec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_dec), " { .. }")) - } - } - - impl AlgorithmName for $name_dec { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_dec)) - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_dec {} - - pub(crate) struct $name_back_enc<'a>(&'a $name); - - impl<'a> BlockSizeUser for $name_back_enc<'a> { - type BlockSize = U16; - } - - impl<'a> ParBlocksSizeUser for $name_back_enc<'a> { - type ParBlocksSize = FixsliceBlocks; - } - - impl<'a> BlockCipherEncBackend for $name_back_enc<'a> { - #[inline(always)] - fn encrypt_block(&self, mut block: InOut<'_, '_, Block>) { - let mut blocks = BatchBlocks::default(); - blocks[0] = block.clone_in().into(); - let res = $fixslice_encrypt(&self.0.keys, &blocks); - *block.get_out() = res[0].into(); - } - - #[inline(always)] - fn encrypt_par_blocks(&self, mut blocks: InOut<'_, '_, BatchBlocks>) { - let res = $fixslice_encrypt(&self.0.keys, blocks.get_in()); - *blocks.get_out() = res; - } - } - - pub(crate) struct $name_back_dec<'a>(&'a $name); - - impl<'a> BlockSizeUser for $name_back_dec<'a> { - type BlockSize = U16; - } - - impl<'a> ParBlocksSizeUser for $name_back_dec<'a> { - type ParBlocksSize = FixsliceBlocks; - } - - impl<'a> BlockCipherDecBackend for $name_back_dec<'a> { - #[inline(always)] - fn decrypt_block(&self, mut block: InOut<'_, '_, Block>) { - let mut blocks = BatchBlocks::default(); - blocks[0] = block.clone_in(); - let res = $fixslice_decrypt(&self.0.keys, &blocks); - *block.get_out() = res[0]; - } - - #[inline(always)] - fn decrypt_par_blocks(&self, mut blocks: InOut<'_, '_, BatchBlocks>) { - let res = $fixslice_decrypt(&self.0.keys, blocks.get_in()); - *blocks.get_out() = res; - } - } - }; -} - -define_aes_impl!( - Aes128, - Aes128Enc, - Aes128Dec, - Aes128BackEnc, - Aes128BackDec, - U16, - FixsliceKeys128, - fixslice::aes128_key_schedule, - fixslice::aes128_decrypt, - fixslice::aes128_encrypt, - "AES-128", -); - -define_aes_impl!( - Aes192, - Aes192Enc, - Aes192Dec, - Aes192BackEnc, - Aes192BackDec, - U24, - FixsliceKeys192, - fixslice::aes192_key_schedule, - fixslice::aes192_decrypt, - fixslice::aes192_encrypt, - "AES-192", -); - -define_aes_impl!( - Aes256, - Aes256Enc, - Aes256Dec, - Aes256BackEnc, - Aes256BackDec, - U32, - FixsliceKeys256, - fixslice::aes256_key_schedule, - fixslice::aes256_decrypt, - fixslice::aes256_encrypt, - "AES-256", -); diff --git a/aes/src/soft/fixslice.rs b/aes/src/soft/fixslice.rs deleted file mode 100644 index e4bd1fbc6..000000000 --- a/aes/src/soft/fixslice.rs +++ /dev/null @@ -1,1842 +0,0 @@ -//! Fixsliced implementations of AES-128, AES-192 and AES-256 -//! adapted from the C implementation. -//! -//! All implementations are fully bitsliced and do not rely on any -//! Look-Up Table (LUT). -//! -//! See the paper at for more details. -//! -//! The machine-word width is abstracted through the [`Word`] trait: the -//! algorithm body is written generically, and the two width-specific [`Word`] -//! impls (for `u32` and `u64`) carry the row-width-dependent mask constants -//! and the `bitslice` / `inv_bitslice` packing routines. The default word -//! type is selected at compile time from `target_pointer_width`. -//! -//! # Author (original C code) -//! -//! Alexandre Adomnicai, Nanyang Technological University, Singapore -//! -//! -//! Originally licensed MIT. Relicensed as Apache 2.0+MIT with permission. - -#![allow(clippy::unreadable_literal)] - -use crate::Block; -use cipher::{ - array::{Array, ArraySize}, - consts::{U2, U4}, -}; -use core::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not, Shl, Shr}; - -/// Width-abstracted machine word holding one row of a bitsliced AES state. -pub(crate) trait Word: - Copy - + Default - + 'static - + BitAnd - + BitAndAssign - + BitOr - + BitOrAssign - + BitXor - + BitXorAssign - + Not - + Shl - + Shr -{ - /// Number of 128-bit blocks bitsliced together in one state. - type Blocks: ArraySize; - - /// Width in bits of one row of the bitsliced state (8 for `u32`, 16 for `u64`). - const ROW_BITS: u32; - - /// Half of `ROW_BITS`. - const HALF_ROW: u32 = Self::ROW_BITS / 2; - /// Quarter of `ROW_BITS`. - const QUARTER_ROW: u32 = Self::ROW_BITS / 4; - - /// Distance in bits to rotate a state row by `(rows, cols)` positions. - #[inline(always)] - fn ror_distance(rows: u32, cols: u32) -> u32 { - rows * Self::ROW_BITS + cols * Self::QUARTER_ROW - } - - /// Rotate right by `n` bits. - fn ror(self, n: u32) -> Self; - - /// Pack the same byte across all 4 rows of the word. - fn uniform_row(b: u8) -> Self; - - /// Place one byte at each of the 4 row positions of the word (row 0 = LSB). - fn pack_rows(r0: u8, r1: u8, r2: u8, r3: u8) -> Self; - - /// Replicate byte `b` across every byte of the word. - fn byte_repeat(b: u8) -> Self; - - /// Pack `Self::Blocks` input blocks into a bitsliced 8-row state slice. - fn bitslice(output: &mut [Self], input: &Array); - - /// Unpack a bitsliced 8-row state slice into `Self::Blocks` output blocks. - fn inv_bitslice(input: &[Self]) -> Array; -} - -/// Width-generic delta-swap pipeline shared by `bitslice` and `inv_bitslice` -/// across every `Word` impl. The same three-pass sequence inverts itself, so -/// `bitslice` and `inv_bitslice` invoke it identically. -/// -/// The diagrams below describe the `u32` case (8-bit rows); for `u64` each -/// bit position widens by one, but the swap structure is unchanged. -#[inline(always)] -fn bitslice_swaps(t: &mut [W; 8]) { - let [t0, t1, t2, t3, t4, t5, t6, t7] = t; - - // Bit Index Swap 5 <-> 0: - // __ __ b0 __ __ __ __ p0 => __ __ p0 __ __ __ __ b0 - let m0 = W::byte_repeat(0x55); - delta_swap_2(t1, t0, 1, m0); - delta_swap_2(t3, t2, 1, m0); - delta_swap_2(t5, t4, 1, m0); - delta_swap_2(t7, t6, 1, m0); - - // Bit Index Swap 6 <-> 1: - // __ c0 __ __ __ __ p1 __ => __ p1 __ __ __ __ c0 __ - let m1 = W::byte_repeat(0x33); - delta_swap_2(t2, t0, 2, m1); - delta_swap_2(t3, t1, 2, m1); - delta_swap_2(t6, t4, 2, m1); - delta_swap_2(t7, t5, 2, m1); - - // Bit Index Swap 7 <-> 2: - // c1 __ __ __ __ p2 __ __ => p2 __ __ __ __ c1 __ __ - let m2 = W::byte_repeat(0x0f); - delta_swap_2(t4, t0, 4, m2); - delta_swap_2(t5, t1, 4, m2); - delta_swap_2(t6, t2, 4, m2); - delta_swap_2(t7, t3, 4, m2); -} - -// ===================================================================== -// Generic type aliases used by the algorithm body -// ===================================================================== - -/// Bitsliced internal state: 8 word-wide rows (256-bit for `u32`, 512-bit for `u64`). -type State = [W; 8]; -/// Input/output batch: `W::Blocks` AES blocks packed together. -type Batch = Array::Blocks>; -/// AES-128 round keys. -type Keys128 = [W; 88]; -/// AES-192 round keys. -type Keys192 = [W; 104]; -/// AES-256 round keys. -type Keys256 = [W; 120]; - -/// Replicate a single 16-byte input block across all slots of a `Batch`. -/// -/// Used by the key schedules, which conceptually call `bitslice(...)` on the -/// same input block several times to fill the bitsliced state. -fn broadcast(block: &[u8]) -> Batch { - debug_assert_eq!(block.len(), 16); - let mut out = Batch::::default(); - for slot in out.iter_mut() { - slot.copy_from_slice(block); - } - out -} - -// ===================================================================== -// Key schedules -// ===================================================================== - -/// Fully bitsliced AES-128 key schedule to match the fully-fixsliced representation. -fn aes128_key_schedule_generic(key: &[u8; 16]) -> Keys128 { - let mut rkeys = [W::default(); 88]; - - W::bitslice(&mut rkeys[..8], &broadcast::(key)); - - let mut rk_off = 0; - for rcon in 0..10 { - memshift32(&mut rkeys, rk_off); - rk_off += 8; - - sub_bytes(&mut rkeys[rk_off..(rk_off + 8)]); - sub_bytes_nots(&mut rkeys[rk_off..(rk_off + 8)]); - - if rcon < 8 { - add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon); - } else { - add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 8); - add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 7); - add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 5); - add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon - 4); - } - - xor_columns(&mut rkeys, rk_off, 8, W::ror_distance(1, 3)); - } - - // Adjust to match fixslicing format - #[cfg(aes_backend_soft = "compact")] - { - for i in (8..88).step_by(16) { - inv_shift_rows_1(&mut rkeys[i..(i + 8)]); - } - } - #[cfg(not(aes_backend_soft = "compact"))] - { - for i in (8..72).step_by(32) { - inv_shift_rows_1(&mut rkeys[i..(i + 8)]); - inv_shift_rows_2(&mut rkeys[(i + 8)..(i + 16)]); - inv_shift_rows_3(&mut rkeys[(i + 16)..(i + 24)]); - } - inv_shift_rows_1(&mut rkeys[72..80]); - } - - // Account for NOTs removed from sub_bytes - for i in 1..11 { - sub_bytes_nots(&mut rkeys[(i * 8)..(i * 8 + 8)]); - } - - rkeys -} - -/// Fully bitsliced AES-192 key schedule to match the fully-fixsliced representation. -fn aes192_key_schedule_generic(key: &[u8; 24]) -> Keys192 { - let mut rkeys = [W::default(); 104]; - let mut tmp = [W::default(); 8]; - - W::bitslice(&mut rkeys[..8], &broadcast::(&key[..16])); - W::bitslice(&mut tmp, &broadcast::(&key[8..])); - - let mut rcon = 0; - let mut rk_off = 8; - - loop { - for i in 0..8 { - rkeys[rk_off + i] = (W::uniform_row(0x0f) & (tmp[i] >> W::HALF_ROW)) - | (W::uniform_row(0xf0) & (rkeys[(rk_off - 8) + i] << W::HALF_ROW)); - } - - sub_bytes(&mut tmp); - sub_bytes_nots(&mut tmp); - - add_round_constant_bit(&mut tmp, rcon); - rcon += 1; - - for i in 0..8 { - let mut ti = rkeys[rk_off + i]; - ti ^= W::uniform_row(0x30) & tmp[i].ror(W::ror_distance(1, 1)); - ti ^= W::uniform_row(0xc0) & (ti << W::QUARTER_ROW); - tmp[i] = ti; - } - rkeys[rk_off..(rk_off + 8)].copy_from_slice(&tmp); - rk_off += 8; - - for i in 0..8 { - let ui = tmp[i]; - let mut ti = (W::uniform_row(0x0f) & (rkeys[(rk_off - 16) + i] >> W::HALF_ROW)) - | (W::uniform_row(0xf0) & (ui << W::HALF_ROW)); - ti ^= W::uniform_row(0x03) & (ui >> (3 * W::QUARTER_ROW)); - tmp[i] = ti - ^ (W::uniform_row(0xfc) & (ti << W::QUARTER_ROW)) - ^ (W::uniform_row(0xf0) & (ti << W::HALF_ROW)) - ^ (W::uniform_row(0xc0) & (ti << (3 * W::QUARTER_ROW))); - } - rkeys[rk_off..(rk_off + 8)].copy_from_slice(&tmp); - rk_off += 8; - - sub_bytes(&mut tmp); - sub_bytes_nots(&mut tmp); - - add_round_constant_bit(&mut tmp, rcon); - rcon += 1; - - for i in 0..8 { - let mut ti = (W::uniform_row(0x0f) & (rkeys[(rk_off - 16) + i] >> W::HALF_ROW)) - | (W::uniform_row(0xf0) & (rkeys[(rk_off - 8) + i] << W::HALF_ROW)); - ti ^= W::uniform_row(0x03) & tmp[i].ror(W::ror_distance(1, 3)); - rkeys[rk_off + i] = ti - ^ (W::uniform_row(0xfc) & (ti << W::QUARTER_ROW)) - ^ (W::uniform_row(0xf0) & (ti << W::HALF_ROW)) - ^ (W::uniform_row(0xc0) & (ti << (3 * W::QUARTER_ROW))); - } - rk_off += 8; - - if rcon >= 8 { - break; - } - - for i in 0..8 { - let ui = rkeys[(rk_off - 8) + i]; - let mut ti = rkeys[(rk_off - 16) + i]; - ti ^= W::uniform_row(0x30) & (ui >> W::QUARTER_ROW); - ti ^= W::uniform_row(0xc0) & (ti << W::QUARTER_ROW); - tmp[i] = ti; - } - } - - // Adjust to match fixslicing format - #[cfg(aes_backend_soft = "compact")] - { - for i in (8..104).step_by(16) { - inv_shift_rows_1(&mut rkeys[i..(i + 8)]); - } - } - #[cfg(not(aes_backend_soft = "compact"))] - { - for i in (0..96).step_by(32) { - inv_shift_rows_1(&mut rkeys[(i + 8)..(i + 16)]); - inv_shift_rows_2(&mut rkeys[(i + 16)..(i + 24)]); - inv_shift_rows_3(&mut rkeys[(i + 24)..(i + 32)]); - } - } - - // Account for NOTs removed from sub_bytes - for i in 1..13 { - sub_bytes_nots(&mut rkeys[(i * 8)..(i * 8 + 8)]); - } - - rkeys -} - -/// Fully bitsliced AES-256 key schedule to match the fully-fixsliced representation. -fn aes256_key_schedule_generic(key: &[u8; 32]) -> Keys256 { - let mut rkeys = [W::default(); 120]; - - W::bitslice(&mut rkeys[..8], &broadcast::(&key[..16])); - W::bitslice(&mut rkeys[8..16], &broadcast::(&key[16..])); - - let mut rk_off = 8; - - let mut rcon = 0; - loop { - memshift32(&mut rkeys, rk_off); - rk_off += 8; - - sub_bytes(&mut rkeys[rk_off..(rk_off + 8)]); - sub_bytes_nots(&mut rkeys[rk_off..(rk_off + 8)]); - - add_round_constant_bit(&mut rkeys[rk_off..(rk_off + 8)], rcon); - xor_columns(&mut rkeys, rk_off, 16, W::ror_distance(1, 3)); - rcon += 1; - - if rcon == 7 { - break; - } - - memshift32(&mut rkeys, rk_off); - rk_off += 8; - - sub_bytes(&mut rkeys[rk_off..(rk_off + 8)]); - sub_bytes_nots(&mut rkeys[rk_off..(rk_off + 8)]); - - xor_columns(&mut rkeys, rk_off, 16, W::ror_distance(0, 3)); - } - - // Adjust to match fixslicing format - #[cfg(aes_backend_soft = "compact")] - { - for i in (8..120).step_by(16) { - inv_shift_rows_1(&mut rkeys[i..(i + 8)]); - } - } - #[cfg(not(aes_backend_soft = "compact"))] - { - for i in (8..104).step_by(32) { - inv_shift_rows_1(&mut rkeys[i..(i + 8)]); - inv_shift_rows_2(&mut rkeys[(i + 8)..(i + 16)]); - inv_shift_rows_3(&mut rkeys[(i + 16)..(i + 24)]); - } - inv_shift_rows_1(&mut rkeys[104..112]); - } - - // Account for NOTs removed from sub_bytes - for i in 1..15 { - sub_bytes_nots(&mut rkeys[(i * 8)..(i * 8 + 8)]); - } - - rkeys -} - -// ===================================================================== -// Encryption / decryption -// ===================================================================== - -/// Fully-fixsliced AES-128 decryption (the InvShiftRows is completely omitted). -/// -/// Decrypts `W::Blocks` blocks in-place and in parallel. -fn aes128_decrypt_generic(rkeys: &Keys128, blocks: &Batch) -> Batch { - let mut state = State::::default(); - - W::bitslice(&mut state, blocks); - - add_round_key(&mut state, &rkeys[80..]); - inv_sub_bytes(&mut state); - - #[cfg(not(aes_backend_soft = "compact"))] - { - inv_shift_rows_2(&mut state); - } - - let mut rk_off = 72; - loop { - #[cfg(aes_backend_soft = "compact")] - { - inv_shift_rows_2(&mut state); - } - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_1(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - if rk_off == 0 { - break; - } - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_0(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - #[cfg(not(aes_backend_soft = "compact"))] - { - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_3(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_2(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - } - } - - add_round_key(&mut state, &rkeys[..8]); - - W::inv_bitslice(&state) -} - -/// Fully-fixsliced AES-128 encryption (the ShiftRows is completely omitted). -/// -/// Encrypts `W::Blocks` blocks in-place and in parallel. -fn aes128_encrypt_generic(rkeys: &Keys128, blocks: &Batch) -> Batch { - let mut state = State::::default(); - - W::bitslice(&mut state, blocks); - - add_round_key(&mut state, &rkeys[..8]); - - let mut rk_off = 8; - loop { - sub_bytes(&mut state); - mix_columns_1(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - - #[cfg(aes_backend_soft = "compact")] - { - shift_rows_2(&mut state); - } - - if rk_off == 80 { - break; - } - - #[cfg(not(aes_backend_soft = "compact"))] - { - sub_bytes(&mut state); - mix_columns_2(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - - sub_bytes(&mut state); - mix_columns_3(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - } - - sub_bytes(&mut state); - mix_columns_0(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - } - - #[cfg(not(aes_backend_soft = "compact"))] - { - shift_rows_2(&mut state); - } - - sub_bytes(&mut state); - add_round_key(&mut state, &rkeys[80..]); - - W::inv_bitslice(&state) -} - -/// Fully-fixsliced AES-192 decryption (the InvShiftRows is completely omitted). -/// -/// Decrypts `W::Blocks` blocks in-place and in parallel. -fn aes192_decrypt_generic(rkeys: &Keys192, blocks: &Batch) -> Batch { - let mut state = State::::default(); - - W::bitslice(&mut state, blocks); - - add_round_key(&mut state, &rkeys[96..]); - inv_sub_bytes(&mut state); - - let mut rk_off = 88; - loop { - #[cfg(aes_backend_soft = "compact")] - { - inv_shift_rows_2(&mut state); - } - #[cfg(not(aes_backend_soft = "compact"))] - { - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_3(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_2(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - } - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_1(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - if rk_off == 0 { - break; - } - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_0(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - } - - add_round_key(&mut state, &rkeys[..8]); - - W::inv_bitslice(&state) -} - -/// Fully-fixsliced AES-192 encryption (the ShiftRows is completely omitted). -/// -/// Encrypts `W::Blocks` blocks in-place and in parallel. -fn aes192_encrypt_generic(rkeys: &Keys192, blocks: &Batch) -> Batch { - let mut state = State::::default(); - - W::bitslice(&mut state, blocks); - - add_round_key(&mut state, &rkeys[..8]); - - let mut rk_off = 8; - loop { - sub_bytes(&mut state); - mix_columns_1(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - - #[cfg(aes_backend_soft = "compact")] - { - shift_rows_2(&mut state); - } - #[cfg(not(aes_backend_soft = "compact"))] - { - sub_bytes(&mut state); - mix_columns_2(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - - sub_bytes(&mut state); - mix_columns_3(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - } - - if rk_off == 96 { - break; - } - - sub_bytes(&mut state); - mix_columns_0(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - } - - sub_bytes(&mut state); - add_round_key(&mut state, &rkeys[96..]); - - W::inv_bitslice(&state) -} - -/// Fully-fixsliced AES-256 decryption (the InvShiftRows is completely omitted). -/// -/// Decrypts `W::Blocks` blocks in-place and in parallel. -fn aes256_decrypt_generic(rkeys: &Keys256, blocks: &Batch) -> Batch { - let mut state = State::::default(); - - W::bitslice(&mut state, blocks); - - add_round_key(&mut state, &rkeys[112..]); - inv_sub_bytes(&mut state); - - #[cfg(not(aes_backend_soft = "compact"))] - { - inv_shift_rows_2(&mut state); - } - - let mut rk_off = 104; - loop { - #[cfg(aes_backend_soft = "compact")] - { - inv_shift_rows_2(&mut state); - } - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_1(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - if rk_off == 0 { - break; - } - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_0(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - #[cfg(not(aes_backend_soft = "compact"))] - { - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_3(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - inv_mix_columns_2(&mut state); - inv_sub_bytes(&mut state); - rk_off -= 8; - } - } - - add_round_key(&mut state, &rkeys[..8]); - - W::inv_bitslice(&state) -} - -/// Fully-fixsliced AES-256 encryption (the ShiftRows is completely omitted). -/// -/// Encrypts `W::Blocks` blocks in-place and in parallel. -fn aes256_encrypt_generic(rkeys: &Keys256, blocks: &Batch) -> Batch { - let mut state = State::::default(); - - W::bitslice(&mut state, blocks); - - add_round_key(&mut state, &rkeys[..8]); - - let mut rk_off = 8; - loop { - sub_bytes(&mut state); - mix_columns_1(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - - #[cfg(aes_backend_soft = "compact")] - { - shift_rows_2(&mut state); - } - - if rk_off == 112 { - break; - } - - #[cfg(not(aes_backend_soft = "compact"))] - { - sub_bytes(&mut state); - mix_columns_2(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - - sub_bytes(&mut state); - mix_columns_3(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - } - - sub_bytes(&mut state); - mix_columns_0(&mut state); - add_round_key(&mut state, &rkeys[rk_off..(rk_off + 8)]); - rk_off += 8; - } - - #[cfg(not(aes_backend_soft = "compact"))] - { - shift_rows_2(&mut state); - } - - sub_bytes(&mut state); - add_round_key(&mut state, &rkeys[112..]); - - W::inv_bitslice(&state) -} - -// ===================================================================== -// S-box and inverse S-box (Boyar-Peralta-Calik) -// ===================================================================== - -/// Note that the 4 bitwise NOT are accounted for here so that it is a true -/// inverse of `sub_bytes`. -fn inv_sub_bytes(state: &mut [W]) { - debug_assert_eq!(state.len(), 8); - - // Scheduled using https://github.com/Ko-/aes-armcortexm/tree/public/scheduler - // Inline "stack" comments reflect suggested stores and loads (ARM Cortex-M3 and M4) - - let u7 = state[0]; - let u6 = state[1]; - let u5 = state[2]; - let u4 = state[3]; - let u3 = state[4]; - let u2 = state[5]; - let u1 = state[6]; - let u0 = state[7]; - - let t23 = u0 ^ u3; - let t8 = u1 ^ t23; - let m2 = t23 & t8; - let t4 = u4 ^ t8; - let t22 = u1 ^ u3; - let t2 = u0 ^ u1; - let t1 = u3 ^ u4; - // t23 -> stack - let t9 = u7 ^ t1; - // t8 -> stack - let m7 = t22 & t9; - // t9 -> stack - let t24 = u4 ^ u7; - // m7 -> stack - let t10 = t2 ^ t24; - // u4 -> stack - let m14 = t2 & t10; - let r5 = u6 ^ u7; - // m2 -> stack - let t3 = t1 ^ r5; - // t2 -> stack - let t13 = t2 ^ r5; - let t19 = t22 ^ r5; - // t3 -> stack - let t17 = u2 ^ t19; - // t4 -> stack - let t25 = u2 ^ t1; - let r13 = u1 ^ u6; - // t25 -> stack - let t20 = t24 ^ r13; - // t17 -> stack - let m9 = t20 & t17; - // t20 -> stack - let r17 = u2 ^ u5; - // t22 -> stack - let t6 = t22 ^ r17; - // t13 -> stack - let m1 = t13 & t6; - let y5 = u0 ^ r17; - let m4 = t19 & y5; - let m5 = m4 ^ m1; - let m17 = m5 ^ t24; - let r18 = u5 ^ u6; - let t27 = t1 ^ r18; - let t15 = t10 ^ t27; - // t6 -> stack - let m11 = t1 & t15; - let m15 = m14 ^ m11; - let m21 = m17 ^ m15; - // t1 -> stack - // t4 <- stack - let m12 = t4 & t27; - let m13 = m12 ^ m11; - let t14 = t10 ^ r18; - let m3 = t14 ^ m1; - // m2 <- stack - let m16 = m3 ^ m2; - let m20 = m16 ^ m13; - // u4 <- stack - let r19 = u2 ^ u4; - let t16 = r13 ^ r19; - // t3 <- stack - let t26 = t3 ^ t16; - let m6 = t3 & t16; - let m8 = t26 ^ m6; - // t10 -> stack - // m7 <- stack - let m18 = m8 ^ m7; - let m22 = m18 ^ m13; - let m25 = m22 & m20; - let m26 = m21 ^ m25; - let m10 = m9 ^ m6; - let m19 = m10 ^ m15; - // t25 <- stack - let m23 = m19 ^ t25; - let m28 = m23 ^ m25; - let m24 = m22 ^ m23; - let m30 = m26 & m24; - let m39 = m23 ^ m30; - let m48 = m39 & y5; - let m57 = m39 & t19; - // m48 -> stack - let m36 = m24 ^ m25; - let m31 = m20 & m23; - let m27 = m20 ^ m21; - let m32 = m27 & m31; - let m29 = m28 & m27; - let m37 = m21 ^ m29; - // m39 -> stack - let m42 = m37 ^ m39; - let m52 = m42 & t15; - // t27 -> stack - // t1 <- stack - let m61 = m42 & t1; - let p0 = m52 ^ m61; - let p16 = m57 ^ m61; - // m57 -> stack - // t20 <- stack - let m60 = m37 & t20; - // p16 -> stack - // t17 <- stack - let m51 = m37 & t17; - let m33 = m27 ^ m25; - let m38 = m32 ^ m33; - let m43 = m37 ^ m38; - let m49 = m43 & t16; - let p6 = m49 ^ m60; - let p13 = m49 ^ m51; - let m58 = m43 & t3; - // t9 <- stack - let m50 = m38 & t9; - // t22 <- stack - let m59 = m38 & t22; - // p6 -> stack - let p1 = m58 ^ m59; - let p7 = p0 ^ p1; - let m34 = m21 & m22; - let m35 = m24 & m34; - let m40 = m35 ^ m36; - let m41 = m38 ^ m40; - let m45 = m42 ^ m41; - // t27 <- stack - let m53 = m45 & t27; - let p8 = m50 ^ m53; - let p23 = p7 ^ p8; - // t4 <- stack - let m62 = m45 & t4; - let p14 = m49 ^ m62; - let s6 = p14 ^ p23; - // t10 <- stack - let m54 = m41 & t10; - let p2 = m54 ^ m62; - let p22 = p2 ^ p7; - let s0 = p13 ^ p22; - let p17 = m58 ^ p2; - let p15 = m54 ^ m59; - // t2 <- stack - let m63 = m41 & t2; - // m39 <- stack - let m44 = m39 ^ m40; - // p17 -> stack - // t6 <- stack - let m46 = m44 & t6; - let p5 = m46 ^ m51; - // p23 -> stack - let p18 = m63 ^ p5; - let p24 = p5 ^ p7; - // m48 <- stack - let p12 = m46 ^ m48; - let s3 = p12 ^ p22; - // t13 <- stack - let m55 = m44 & t13; - let p9 = m55 ^ m63; - // p16 <- stack - let s7 = p9 ^ p16; - // t8 <- stack - let m47 = m40 & t8; - let p3 = m47 ^ m50; - let p19 = p2 ^ p3; - let s5 = p19 ^ p24; - let p11 = p0 ^ p3; - let p26 = p9 ^ p11; - // t23 <- stack - let m56 = m40 & t23; - let p4 = m48 ^ m56; - // p6 <- stack - let p20 = p4 ^ p6; - let p29 = p15 ^ p20; - let s1 = p26 ^ p29; - // m57 <- stack - let p10 = m57 ^ p4; - let p27 = p10 ^ p18; - // p23 <- stack - let s4 = p23 ^ p27; - let p25 = p6 ^ p10; - let p28 = p11 ^ p25; - // p17 <- stack - let s2 = p17 ^ p28; - - state[0] = s7; - state[1] = s6; - state[2] = s5; - state[3] = s4; - state[4] = s3; - state[5] = s2; - state[6] = s1; - state[7] = s0; -} - -/// Bitsliced implementation of the AES Sbox based on Boyar, Peralta and Calik. -/// -/// See: -/// -/// Note that the 4 bitwise NOT are moved to the key schedule. -fn sub_bytes(state: &mut [W]) { - debug_assert_eq!(state.len(), 8); - - // Scheduled using https://github.com/Ko-/aes-armcortexm/tree/public/scheduler - // Inline "stack" comments reflect suggested stores and loads (ARM Cortex-M3 and M4) - - let u7 = state[0]; - let u6 = state[1]; - let u5 = state[2]; - let u4 = state[3]; - let u3 = state[4]; - let u2 = state[5]; - let u1 = state[6]; - let u0 = state[7]; - - let y14 = u3 ^ u5; - let y13 = u0 ^ u6; - let y12 = y13 ^ y14; - let t1 = u4 ^ y12; - let y15 = t1 ^ u5; - let t2 = y12 & y15; - let y6 = y15 ^ u7; - let y20 = t1 ^ u1; - // y12 -> stack - let y9 = u0 ^ u3; - // y20 -> stack - let y11 = y20 ^ y9; - // y9 -> stack - let t12 = y9 & y11; - // y6 -> stack - let y7 = u7 ^ y11; - let y8 = u0 ^ u5; - let t0 = u1 ^ u2; - let y10 = y15 ^ t0; - // y15 -> stack - let y17 = y10 ^ y11; - // y14 -> stack - let t13 = y14 & y17; - let t14 = t13 ^ t12; - // y17 -> stack - let y19 = y10 ^ y8; - // y10 -> stack - let t15 = y8 & y10; - let t16 = t15 ^ t12; - let y16 = t0 ^ y11; - // y11 -> stack - let y21 = y13 ^ y16; - // y13 -> stack - let t7 = y13 & y16; - // y16 -> stack - let y18 = u0 ^ y16; - let y1 = t0 ^ u7; - let y4 = y1 ^ u3; - // u7 -> stack - let t5 = y4 & u7; - let t6 = t5 ^ t2; - let t18 = t6 ^ t16; - let t22 = t18 ^ y19; - let y2 = y1 ^ u0; - let t10 = y2 & y7; - let t11 = t10 ^ t7; - let t20 = t11 ^ t16; - let t24 = t20 ^ y18; - let y5 = y1 ^ u6; - let t8 = y5 & y1; - let t9 = t8 ^ t7; - let t19 = t9 ^ t14; - let t23 = t19 ^ y21; - let y3 = y5 ^ y8; - // y6 <- stack - let t3 = y3 & y6; - let t4 = t3 ^ t2; - // y20 <- stack - let t17 = t4 ^ y20; - let t21 = t17 ^ t14; - let t26 = t21 & t23; - let t27 = t24 ^ t26; - let t31 = t22 ^ t26; - let t25 = t21 ^ t22; - // y4 -> stack - let t28 = t25 & t27; - let t29 = t28 ^ t22; - let z14 = t29 & y2; - let z5 = t29 & y7; - let t30 = t23 ^ t24; - let t32 = t31 & t30; - let t33 = t32 ^ t24; - let t35 = t27 ^ t33; - let t36 = t24 & t35; - let t38 = t27 ^ t36; - let t39 = t29 & t38; - let t40 = t25 ^ t39; - let t43 = t29 ^ t40; - // y16 <- stack - let z3 = t43 & y16; - let tc12 = z3 ^ z5; - // tc12 -> stack - // y13 <- stack - let z12 = t43 & y13; - let z13 = t40 & y5; - let z4 = t40 & y1; - let tc6 = z3 ^ z4; - let t34 = t23 ^ t33; - let t37 = t36 ^ t34; - let t41 = t40 ^ t37; - // y10 <- stack - let z8 = t41 & y10; - let z17 = t41 & y8; - let t44 = t33 ^ t37; - // y15 <- stack - let z0 = t44 & y15; - // z17 -> stack - // y12 <- stack - let z9 = t44 & y12; - let z10 = t37 & y3; - let z1 = t37 & y6; - let tc5 = z1 ^ z0; - let tc11 = tc6 ^ tc5; - // y4 <- stack - let z11 = t33 & y4; - let t42 = t29 ^ t33; - let t45 = t42 ^ t41; - // y17 <- stack - let z7 = t45 & y17; - let tc8 = z7 ^ tc6; - // y14 <- stack - let z16 = t45 & y14; - // y11 <- stack - let z6 = t42 & y11; - let tc16 = z6 ^ tc8; - // z14 -> stack - // y9 <- stack - let z15 = t42 & y9; - let tc20 = z15 ^ tc16; - let tc1 = z15 ^ z16; - let tc2 = z10 ^ tc1; - let tc21 = tc2 ^ z11; - let tc3 = z9 ^ tc2; - let s0 = tc3 ^ tc16; - let s3 = tc3 ^ tc11; - let s1 = s3 ^ tc16; - let tc13 = z13 ^ tc1; - // u7 <- stack - let z2 = t33 & u7; - let tc4 = z0 ^ z2; - let tc7 = z12 ^ tc4; - let tc9 = z8 ^ tc7; - let tc10 = tc8 ^ tc9; - // z14 <- stack - let tc17 = z14 ^ tc10; - let s5 = tc21 ^ tc17; - let tc26 = tc17 ^ tc20; - // z17 <- stack - let s2 = tc26 ^ z17; - // tc12 <- stack - let tc14 = tc4 ^ tc12; - let tc18 = tc13 ^ tc14; - let s6 = tc10 ^ tc18; - let s7 = z12 ^ tc18; - let s4 = tc14 ^ s3; - - state[0] = s7; - state[1] = s6; - state[2] = s5; - state[3] = s4; - state[4] = s3; - state[5] = s2; - state[6] = s1; - state[7] = s0; -} - -/// NOT operations that are omitted in S-box. -#[inline] -fn sub_bytes_nots(state: &mut [W]) { - debug_assert_eq!(state.len(), 8); - state[0] = !state[0]; - state[1] = !state[1]; - state[5] = !state[5]; - state[6] = !state[6]; -} - -// ===================================================================== -// MixColumns -// ===================================================================== - -/// Computation of the MixColumns transformation in the fixsliced representation, -/// with different rotations used according to the round number mod 4. -/// -/// Based on Käsper-Schwabe, similar to https://github.com/Ko-/aes-armcortexm. -macro_rules! define_mix_columns { - ( - $name:ident, - $name_inv:ident, - $first_rotate:path, - $second_rotate:path - ) => { - #[rustfmt::skip] - fn $name(state: &mut State) { - let (a0, a1, a2, a3, a4, a5, a6, a7) = ( - state[0], state[1], state[2], state[3], state[4], state[5], state[6], state[7] - ); - let (b0, b1, b2, b3, b4, b5, b6, b7) = ( - $first_rotate(a0), - $first_rotate(a1), - $first_rotate(a2), - $first_rotate(a3), - $first_rotate(a4), - $first_rotate(a5), - $first_rotate(a6), - $first_rotate(a7), - ); - let (c0, c1, c2, c3, c4, c5, c6, c7) = ( - a0 ^ b0, - a1 ^ b1, - a2 ^ b2, - a3 ^ b3, - a4 ^ b4, - a5 ^ b5, - a6 ^ b6, - a7 ^ b7, - ); - state[0] = b0 ^ c7 ^ $second_rotate(c0); - state[1] = b1 ^ c0 ^ c7 ^ $second_rotate(c1); - state[2] = b2 ^ c1 ^ $second_rotate(c2); - state[3] = b3 ^ c2 ^ c7 ^ $second_rotate(c3); - state[4] = b4 ^ c3 ^ c7 ^ $second_rotate(c4); - state[5] = b5 ^ c4 ^ $second_rotate(c5); - state[6] = b6 ^ c5 ^ $second_rotate(c6); - state[7] = b7 ^ c6 ^ $second_rotate(c7); - } - - #[rustfmt::skip] - fn $name_inv(state: &mut State) { - let (a0, a1, a2, a3, a4, a5, a6, a7) = ( - state[0], state[1], state[2], state[3], state[4], state[5], state[6], state[7] - ); - let (b0, b1, b2, b3, b4, b5, b6, b7) = ( - $first_rotate(a0), - $first_rotate(a1), - $first_rotate(a2), - $first_rotate(a3), - $first_rotate(a4), - $first_rotate(a5), - $first_rotate(a6), - $first_rotate(a7), - ); - let (c0, c1, c2, c3, c4, c5, c6, c7) = ( - a0 ^ b0, - a1 ^ b1, - a2 ^ b2, - a3 ^ b3, - a4 ^ b4, - a5 ^ b5, - a6 ^ b6, - a7 ^ b7, - ); - let (d0, d1, d2, d3, d4, d5, d6, d7) = ( - a0 ^ c7, - a1 ^ c0 ^ c7, - a2 ^ c1, - a3 ^ c2 ^ c7, - a4 ^ c3 ^ c7, - a5 ^ c4, - a6 ^ c5, - a7 ^ c6, - ); - let (e0, e1, e2, e3, e4, e5, e6, e7) = ( - c0 ^ d6, - c1 ^ d6 ^ d7, - c2 ^ d0 ^ d7, - c3 ^ d1 ^ d6, - c4 ^ d2 ^ d6 ^ d7, - c5 ^ d3 ^ d7, - c6 ^ d4, - c7 ^ d5, - ); - state[0] = d0 ^ e0 ^ $second_rotate(e0); - state[1] = d1 ^ e1 ^ $second_rotate(e1); - state[2] = d2 ^ e2 ^ $second_rotate(e2); - state[3] = d3 ^ e3 ^ $second_rotate(e3); - state[4] = d4 ^ e4 ^ $second_rotate(e4); - state[5] = d5 ^ e5 ^ $second_rotate(e5); - state[6] = d6 ^ e6 ^ $second_rotate(e6); - state[7] = d7 ^ e7 ^ $second_rotate(e7); - } - }; -} - -define_mix_columns!( - mix_columns_0, - inv_mix_columns_0, - rotate_rows_1, - rotate_rows_2 -); - -define_mix_columns!( - mix_columns_1, - inv_mix_columns_1, - rotate_rows_and_columns_1_1, - rotate_rows_and_columns_2_2 -); - -#[cfg(not(aes_backend_soft = "compact"))] -define_mix_columns!( - mix_columns_2, - inv_mix_columns_2, - rotate_rows_and_columns_1_2, - rotate_rows_2 -); - -#[cfg(not(aes_backend_soft = "compact"))] -define_mix_columns!( - mix_columns_3, - inv_mix_columns_3, - rotate_rows_and_columns_1_3, - rotate_rows_and_columns_2_2 -); - -// ===================================================================== -// Delta swaps and ShiftRows family -// ===================================================================== - -#[inline] -fn delta_swap_1(a: &mut W, shift: u32, mask: W) { - let t = (*a ^ ((*a) >> shift)) & mask; - *a ^= t ^ (t << shift); -} - -#[inline] -fn delta_swap_2(a: &mut W, b: &mut W, shift: u32, mask: W) { - let t = (*a ^ ((*b) >> shift)) & mask; - *a ^= t; - *b ^= t << shift; -} - -/// Applies ShiftRows once on an AES state (or key). -#[cfg(any(not(aes_backend_soft = "compact"), feature = "hazmat"))] -#[inline] -fn shift_rows_1(state: &mut [W]) { - debug_assert_eq!(state.len(), 8); - for x in state.iter_mut() { - delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x03, 0x0f, 0x0c)); - delta_swap_1(x, W::QUARTER_ROW, W::pack_rows(0x00, 0x33, 0x00, 0x33)); - } -} - -/// Applies ShiftRows twice on an AES state (or key). -#[inline] -fn shift_rows_2(state: &mut [W]) { - debug_assert_eq!(state.len(), 8); - for x in state.iter_mut() { - delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x0f, 0x00, 0x0f)); - } -} - -/// Applies ShiftRows three times on an AES state (or key). -#[inline] -fn shift_rows_3(state: &mut [W]) { - debug_assert_eq!(state.len(), 8); - for x in state.iter_mut() { - delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x0c, 0x0f, 0x03)); - delta_swap_1(x, W::QUARTER_ROW, W::pack_rows(0x00, 0x33, 0x00, 0x33)); - } -} - -#[inline(always)] -fn inv_shift_rows_1(state: &mut [W]) { - shift_rows_3(state); -} - -#[inline(always)] -fn inv_shift_rows_2(state: &mut [W]) { - shift_rows_2(state); -} - -#[cfg(not(aes_backend_soft = "compact"))] -#[inline(always)] -fn inv_shift_rows_3(state: &mut [W]) { - shift_rows_1(state); -} - -// ===================================================================== -// Key-schedule helpers, AddRoundKey, AddRoundConstant -// ===================================================================== - -/// XOR the columns after the S-box during the key schedule round function. -/// -/// The `idx_xor` parameter refers to the index of the previous round key -/// involved in the XOR computation (should be 8 and 16 for AES-128 and AES-256, -/// respectively). -/// -/// The `idx_ror` parameter refers to the rotation value, which varies between the -/// different key schedules. -fn xor_columns(rkeys: &mut [W], offset: usize, idx_xor: usize, idx_ror: u32) { - for i in 0..8 { - let off_i = offset + i; - let rk = rkeys[off_i - idx_xor] ^ (W::uniform_row(0x03) & rkeys[off_i].ror(idx_ror)); - rkeys[off_i] = rk - ^ (W::uniform_row(0xfc) & (rk << W::QUARTER_ROW)) - ^ (W::uniform_row(0xf0) & (rk << W::HALF_ROW)) - ^ (W::uniform_row(0xc0) & (rk << (3 * W::QUARTER_ROW))); - } -} - -/// Copy 32-bytes within the provided slice to an 8-byte offset. -fn memshift32(buffer: &mut [W], src_offset: usize) { - debug_assert_eq!(src_offset % 8, 0); - - let dst_offset = src_offset + 8; - debug_assert!(dst_offset + 8 <= buffer.len()); - - for i in (0..8).rev() { - buffer[dst_offset + i] = buffer[src_offset + i]; - } -} - -/// XOR the round key into the internal state. The round keys are expected -/// to be pre-computed and packed in the fixsliced representation. -#[inline] -fn add_round_key(state: &mut State, rkey: &[W]) { - debug_assert_eq!(rkey.len(), 8); - for (a, b) in state.iter_mut().zip(rkey) { - *a ^= *b; - } -} - -#[inline(always)] -fn add_round_constant_bit(state: &mut [W], bit: usize) { - state[bit] ^= W::pack_rows(0x00, 0xc0, 0x00, 0x00); -} - -// ===================================================================== -// Row/column rotations (selected by mix_columns_N round-number variants) -// ===================================================================== - -#[inline(always)] -fn rotate_rows_1(x: W) -> W { - x.ror(W::ror_distance(1, 0)) -} - -#[inline(always)] -fn rotate_rows_2(x: W) -> W { - x.ror(W::ror_distance(2, 0)) -} - -#[inline(always)] -#[rustfmt::skip] -fn rotate_rows_and_columns_1_1(x: W) -> W { - (x.ror(W::ror_distance(1, 1)) & W::uniform_row(0x3f)) | - (x.ror(W::ror_distance(0, 1)) & W::uniform_row(0xc0)) -} - -#[cfg(not(aes_backend_soft = "compact"))] -#[inline(always)] -#[rustfmt::skip] -fn rotate_rows_and_columns_1_2(x: W) -> W { - (x.ror(W::ror_distance(1, 2)) & W::uniform_row(0x0f)) | - (x.ror(W::ror_distance(0, 2)) & W::uniform_row(0xf0)) -} - -#[cfg(not(aes_backend_soft = "compact"))] -#[inline(always)] -#[rustfmt::skip] -fn rotate_rows_and_columns_1_3(x: W) -> W { - (x.ror(W::ror_distance(1, 3)) & W::uniform_row(0x03)) | - (x.ror(W::ror_distance(0, 3)) & W::uniform_row(0xfc)) -} - -#[inline(always)] -#[rustfmt::skip] -fn rotate_rows_and_columns_2_2(x: W) -> W { - (x.ror(W::ror_distance(2, 2)) & W::uniform_row(0x0f)) | - (x.ror(W::ror_distance(1, 2)) & W::uniform_row(0xf0)) -} - -// ===================================================================== -// Word impls — the only width-specific code -// ===================================================================== - -impl Word for u32 { - type Blocks = U2; - - const ROW_BITS: u32 = 8; - - #[inline(always)] - fn ror(self, n: u32) -> u32 { - self.rotate_right(n) - } - - #[inline(always)] - fn uniform_row(b: u8) -> u32 { - (b as u32) * 0x01010101 - } - - #[inline(always)] - fn pack_rows(r0: u8, r1: u8, r2: u8, r3: u8) -> u32 { - (r0 as u32) | ((r1 as u32) << 8) | ((r2 as u32) << 16) | ((r3 as u32) << 24) - } - - #[inline(always)] - fn byte_repeat(b: u8) -> u32 { - (b as u32) * 0x01010101 - } - - /// Bitslice two 128-bit input blocks into a 256-bit internal state. - fn bitslice(output: &mut [u32], input: &Array) { - debug_assert_eq!(output.len(), 8); - let input0 = input[0].as_slice(); - let input1 = input[1].as_slice(); - - // Bitslicing is a bit index manipulation. 256 bits of data means each bit is positioned at - // an 8-bit index. AES data is 2 blocks, each one a 4x4 column-major matrix of bytes, so the - // index is initially ([b]lock, [c]olumn, [r]ow, [p]osition): - // b0 c1 c0 r1 r0 p2 p1 p0 - // - // The desired bitsliced data groups first by bit position, then row, column, block: - // p2 p1 p0 r1 r0 c1 c0 b0 - - // Interleave the columns on input (note the order of input) - // b0 c1 c0 __ __ __ __ __ => c1 c0 b0 __ __ __ __ __ - let mut t = [ - u32::from_le_bytes(input0[0x00..0x04].try_into().unwrap()), - u32::from_le_bytes(input1[0x00..0x04].try_into().unwrap()), - u32::from_le_bytes(input0[0x04..0x08].try_into().unwrap()), - u32::from_le_bytes(input1[0x04..0x08].try_into().unwrap()), - u32::from_le_bytes(input0[0x08..0x0c].try_into().unwrap()), - u32::from_le_bytes(input1[0x08..0x0c].try_into().unwrap()), - u32::from_le_bytes(input0[0x0c..0x10].try_into().unwrap()), - u32::from_le_bytes(input1[0x0c..0x10].try_into().unwrap()), - ]; - - bitslice_swaps(&mut t); - - // Final bitsliced bit index, as desired: - // p2 p1 p0 r1 r0 c1 c0 b0 - output[..8].copy_from_slice(&t); - } - - /// Un-bitslice a 256-bit internal state into two 128-bit blocks. - fn inv_bitslice(input: &[u32]) -> Array { - debug_assert_eq!(input.len(), 8); - - // Unbitslicing is a bit index manipulation. 256 bits of data means each bit is positioned - // at an 8-bit index. AES data is 2 blocks, each one a 4x4 column-major matrix of bytes, so - // the desired index for the output is ([b]lock, [c]olumn, [r]ow, [p]osition): - // b0 c1 c0 r1 r0 p2 p1 p0 - // - // The initially bitsliced data groups first by bit position, then row, column, block: - // p2 p1 p0 r1 r0 c1 c0 b0 - - let mut t = [ - input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7], - ]; - - bitslice_swaps(&mut t); - - let mut output = Array::::default(); - // De-interleave the columns on output (note the order of output) - // c1 c0 b0 __ __ __ __ __ => b0 c1 c0 __ __ __ __ __ - output[0][0x00..0x04].copy_from_slice(&t[0].to_le_bytes()); - output[0][0x04..0x08].copy_from_slice(&t[2].to_le_bytes()); - output[0][0x08..0x0c].copy_from_slice(&t[4].to_le_bytes()); - output[0][0x0c..0x10].copy_from_slice(&t[6].to_le_bytes()); - output[1][0x00..0x04].copy_from_slice(&t[1].to_le_bytes()); - output[1][0x04..0x08].copy_from_slice(&t[3].to_le_bytes()); - output[1][0x08..0x0c].copy_from_slice(&t[5].to_le_bytes()); - output[1][0x0c..0x10].copy_from_slice(&t[7].to_le_bytes()); - - // Final AES bit index, as desired: - // b0 c1 c0 r1 r0 p2 p1 p0 - output - } -} - -/// Expand an 8-bit row pattern to a 16-bit row pattern by doubling each bit: -/// input bit `i` becomes output bits `2i` and `2i+1`. Branchless SWAR so LLVM -/// folds it to a single 16-bit immediate when `b` is a constant. -#[inline(always)] -const fn double_bits(b: u8) -> u16 { - let x = b as u16; - // Spread the 8 bits of x to even positions 0,2,4,6,8,10,12,14. - let x = (x | (x << 4)) & 0x0f0f; - let x = (x | (x << 2)) & 0x3333; - let x = (x | (x << 1)) & 0x5555; - // Duplicate each spread bit to its adjacent odd position. - x | (x << 1) -} - -impl Word for u64 { - type Blocks = U4; - - const ROW_BITS: u32 = 16; - - #[inline(always)] - fn ror(self, n: u32) -> u64 { - self.rotate_right(n) - } - - #[inline(always)] - fn uniform_row(b: u8) -> u64 { - (double_bits(b) as u64) * 0x0001_0001_0001_0001 - } - - #[inline(always)] - fn pack_rows(r0: u8, r1: u8, r2: u8, r3: u8) -> u64 { - (double_bits(r0) as u64) - | ((double_bits(r1) as u64) << 16) - | ((double_bits(r2) as u64) << 32) - | ((double_bits(r3) as u64) << 48) - } - - #[inline(always)] - fn byte_repeat(b: u8) -> u64 { - (b as u64) * 0x0101010101010101 - } - - /// Bitslice four 128-bit input blocks into a 512-bit internal state. - fn bitslice(output: &mut [u64], input: &Array) { - debug_assert_eq!(output.len(), 8); - - // Bitslicing is a bit index manipulation. 512 bits of data means each bit is positioned at - // a 9-bit index. AES data is 4 blocks, each one a 4x4 column-major matrix of bytes, so the - // index is initially ([b]lock, [c]olumn, [r]ow, [p]osition): - // b1 b0 c1 c0 r1 r0 p2 p1 p0 - // - // The desired bitsliced data groups first by bit position, then row, column, block: - // p2 p1 p0 r1 r0 c1 c0 b1 b0 - - #[rustfmt::skip] - fn read_reordered(input: &[u8]) -> u64 { - (u64::from(input[0x0]) ) | - (u64::from(input[0x1]) << 0x10) | - (u64::from(input[0x2]) << 0x20) | - (u64::from(input[0x3]) << 0x30) | - (u64::from(input[0x8]) << 0x08) | - (u64::from(input[0x9]) << 0x18) | - (u64::from(input[0xa]) << 0x28) | - (u64::from(input[0xb]) << 0x38) - } - - // Reorder each block's bytes on input - // __ __ c1 c0 r1 r0 __ __ __ => __ __ c0 r1 r0 c1 __ __ __ - // Reorder by relabeling (note the order of input) - // b1 b0 c0 __ __ __ __ __ __ => c0 b1 b0 __ __ __ __ __ __ - let mut t = [ - read_reordered(&input[0][0x00..0x0c]), - read_reordered(&input[1][0x00..0x0c]), - read_reordered(&input[2][0x00..0x0c]), - read_reordered(&input[3][0x00..0x0c]), - read_reordered(&input[0][0x04..0x10]), - read_reordered(&input[1][0x04..0x10]), - read_reordered(&input[2][0x04..0x10]), - read_reordered(&input[3][0x04..0x10]), - ]; - - bitslice_swaps(&mut t); - - // Final bitsliced bit index, as desired: - // p2 p1 p0 r1 r0 c1 c0 b1 b0 - output[..8].copy_from_slice(&t); - } - - /// Un-bitslice a 512-bit internal state into four 128-bit blocks. - fn inv_bitslice(input: &[u64]) -> Array { - debug_assert_eq!(input.len(), 8); - - // Unbitslicing is a bit index manipulation. 512 bits of data means each bit is positioned - // at a 9-bit index. AES data is 4 blocks, each one a 4x4 column-major matrix of bytes, so - // the desired index for the output is ([b]lock, [c]olumn, [r]ow, [p]osition): - // b1 b0 c1 c0 r1 r0 p2 p1 p0 - // - // The initially bitsliced data groups first by bit position, then row, column, block: - // p2 p1 p0 r1 r0 c1 c0 b1 b0 - - let mut t = [ - input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7], - ]; - - bitslice_swaps(&mut t); - - #[rustfmt::skip] - fn write_reordered(columns: u64, output: &mut [u8]) { - output[0x0] = (columns ) as u8; - output[0x1] = (columns >> 0x10) as u8; - output[0x2] = (columns >> 0x20) as u8; - output[0x3] = (columns >> 0x30) as u8; - output[0x8] = (columns >> 0x08) as u8; - output[0x9] = (columns >> 0x18) as u8; - output[0xa] = (columns >> 0x28) as u8; - output[0xb] = (columns >> 0x38) as u8; - } - - let mut output = Array::::default(); - // Reorder by relabeling (note the order of output) - // c0 b1 b0 __ __ __ __ __ __ => b1 b0 c0 __ __ __ __ __ __ - // Reorder each block's bytes on output - // __ __ c0 r1 r0 c1 __ __ __ => __ __ c1 c0 r1 r0 __ __ __ - write_reordered(t[0], &mut output[0][0x00..0x0c]); - write_reordered(t[4], &mut output[0][0x04..0x10]); - write_reordered(t[1], &mut output[1][0x00..0x0c]); - write_reordered(t[5], &mut output[1][0x04..0x10]); - write_reordered(t[2], &mut output[2][0x00..0x0c]); - write_reordered(t[6], &mut output[2][0x04..0x10]); - write_reordered(t[3], &mut output[3][0x00..0x0c]); - write_reordered(t[7], &mut output[3][0x04..0x10]); - - // Final AES bit index, as desired: - // b1 b0 c1 c0 r1 r0 p2 p1 p0 - output - } -} - -// ===================================================================== -// Concrete re-exports consumed by `soft.rs` -// -// The `Word` impl used for this target is selected at compile time from -// `target_pointer_width`: `u32` on 16/32-bit targets, `u64` on 64-bit. -// ===================================================================== - -cpubits::cpubits! { - 16 | 32 => { - type NativeWord = u32; - } - 64 => { - type NativeWord = u64; - } -} - -/// AES block batch size for this implementation. -pub(crate) type FixsliceBlocks = ::Blocks; - -pub(crate) type BatchBlocks = Batch; - -/// AES-128 round keys. -pub(crate) type FixsliceKeys128 = Keys128; - -/// AES-192 round keys. -pub(crate) type FixsliceKeys192 = Keys192; - -/// AES-256 round keys. -pub(crate) type FixsliceKeys256 = Keys256; - -#[inline] -pub(crate) fn aes128_key_schedule(key: &[u8; 16]) -> FixsliceKeys128 { - aes128_key_schedule_generic::(key) -} - -#[inline] -pub(crate) fn aes192_key_schedule(key: &[u8; 24]) -> FixsliceKeys192 { - aes192_key_schedule_generic::(key) -} - -#[inline] -pub(crate) fn aes256_key_schedule(key: &[u8; 32]) -> FixsliceKeys256 { - aes256_key_schedule_generic::(key) -} - -#[inline] -pub(crate) fn aes128_encrypt(rkeys: &FixsliceKeys128, blocks: &BatchBlocks) -> BatchBlocks { - aes128_encrypt_generic::(rkeys, blocks) -} - -#[inline] -pub(crate) fn aes128_decrypt(rkeys: &FixsliceKeys128, blocks: &BatchBlocks) -> BatchBlocks { - aes128_decrypt_generic::(rkeys, blocks) -} - -#[inline] -pub(crate) fn aes192_encrypt(rkeys: &FixsliceKeys192, blocks: &BatchBlocks) -> BatchBlocks { - aes192_encrypt_generic::(rkeys, blocks) -} - -#[inline] -pub(crate) fn aes192_decrypt(rkeys: &FixsliceKeys192, blocks: &BatchBlocks) -> BatchBlocks { - aes192_decrypt_generic::(rkeys, blocks) -} - -#[inline] -pub(crate) fn aes256_encrypt(rkeys: &FixsliceKeys256, blocks: &BatchBlocks) -> BatchBlocks { - aes256_encrypt_generic::(rkeys, blocks) -} - -#[inline] -pub(crate) fn aes256_decrypt(rkeys: &FixsliceKeys256, blocks: &BatchBlocks) -> BatchBlocks { - aes256_decrypt_generic::(rkeys, blocks) -} - -// ===================================================================== -// Hazmat -// ===================================================================== - -/// Low-level "hazmat" AES functions. -/// -/// Note: this isn't actually used in the `Aes128`/`Aes192`/`Aes256` -/// implementations in this crate, but instead provides raw access to -/// the AES round function gated under the `hazmat` crate feature. -#[cfg(feature = "hazmat")] -pub(crate) mod hazmat { - use super::{ - Batch, NativeWord, State, Word, broadcast, inv_bitslice_one, inv_mix_columns_0, - inv_shift_rows_1, inv_sub_bytes, mix_columns_0, shift_rows_1, sub_bytes, sub_bytes_nots, - }; - use crate::hazmat::{Block, Block8}; - use cipher::typenum::Unsigned; - - /// XOR the `src` block into the `dst` block in-place. - fn xor_in_place(dst: &mut Block, src: &Block) { - for (a, b) in dst.iter_mut().zip(src.as_slice()) { - *a ^= *b; - } - } - - fn cipher_round_generic(block: &mut Block, round_key: &Block) { - let mut state = State::::default(); - W::bitslice(&mut state, &broadcast::(block.as_slice())); - sub_bytes(&mut state); - sub_bytes_nots(&mut state); - shift_rows_1(&mut state); - mix_columns_0(&mut state); - inv_bitslice_one(block, &state); - xor_in_place(block, round_key); - } - - fn cipher_round_par_generic(blocks: &mut Block8, round_keys: &Block8) { - let blocks_per_batch = <::Blocks>::USIZE; - for (chunk, keys) in blocks - .chunks_exact_mut(blocks_per_batch) - .zip(round_keys.chunks_exact(blocks_per_batch)) - { - let mut state = State::::default(); - let mut batch = Batch::::default(); - for (slot, blk) in batch.iter_mut().zip(chunk.iter()) { - slot.copy_from_slice(blk.as_slice()); - } - W::bitslice(&mut state, &batch); - sub_bytes(&mut state); - sub_bytes_nots(&mut state); - shift_rows_1(&mut state); - mix_columns_0(&mut state); - let res = W::inv_bitslice(&state); - - for i in 0..blocks_per_batch { - chunk[i] = res[i]; - xor_in_place(&mut chunk[i], &keys[i]); - } - } - } - - fn equiv_inv_cipher_round_generic(block: &mut Block, round_key: &Block) { - let mut state = State::::default(); - W::bitslice(&mut state, &broadcast::(block.as_slice())); - sub_bytes_nots(&mut state); - inv_sub_bytes(&mut state); - inv_shift_rows_1(&mut state); - inv_mix_columns_0(&mut state); - inv_bitslice_one(block, &state); - xor_in_place(block, round_key); - } - - fn equiv_inv_cipher_round_par_generic(blocks: &mut Block8, round_keys: &Block8) { - let blocks_per_batch = <::Blocks>::USIZE; - for (chunk, keys) in blocks - .chunks_exact_mut(blocks_per_batch) - .zip(round_keys.chunks_exact(blocks_per_batch)) - { - let mut state = State::::default(); - let mut batch = Batch::::default(); - for (slot, blk) in batch.iter_mut().zip(chunk.iter()) { - slot.copy_from_slice(blk.as_slice()); - } - W::bitslice(&mut state, &batch); - sub_bytes_nots(&mut state); - inv_sub_bytes(&mut state); - inv_shift_rows_1(&mut state); - inv_mix_columns_0(&mut state); - let res = W::inv_bitslice(&state); - - for i in 0..blocks_per_batch { - chunk[i] = res[i]; - xor_in_place(&mut chunk[i], &keys[i]); - } - } - } - - fn mix_columns_generic(block: &mut Block) { - let mut state = State::::default(); - W::bitslice(&mut state, &broadcast::(block.as_slice())); - mix_columns_0(&mut state); - inv_bitslice_one(block, &state); - } - - fn inv_mix_columns_generic(block: &mut Block) { - let mut state = State::::default(); - W::bitslice(&mut state, &broadcast::(block.as_slice())); - inv_mix_columns_0(&mut state); - inv_bitslice_one(block, &state); - } - - /// AES cipher (encrypt) round function. - #[inline] - pub(crate) fn cipher_round(block: &mut Block, round_key: &Block) { - cipher_round_generic::(block, round_key) - } - - /// AES cipher (encrypt) round function: parallel version. - #[inline] - pub(crate) fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { - cipher_round_par_generic::(blocks, round_keys) - } - - /// AES cipher (encrypt) inverse round function. - #[inline] - pub(crate) fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { - equiv_inv_cipher_round_generic::(block, round_key) - } - - /// AES cipher (encrypt) inverse round function: parallel version. - #[inline] - pub(crate) fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { - equiv_inv_cipher_round_par_generic::(blocks, round_keys) - } - - /// AES mix columns function. - #[inline] - pub(crate) fn mix_columns(block: &mut Block) { - mix_columns_generic::(block) - } - - /// AES inverse mix columns function. - #[inline] - pub(crate) fn inv_mix_columns(block: &mut Block) { - inv_mix_columns_generic::(block) - } -} - -/// Perform an inverse bitslice operation, extracting a single block. -#[cfg(feature = "hazmat")] -#[inline] -fn inv_bitslice_one(block: &mut Block, state: &State) { - let out = W::inv_bitslice(state); - block.copy_from_slice(out[0].as_slice()); -} diff --git a/aes/src/x86.rs b/aes/src/x86.rs deleted file mode 100644 index 901927531..000000000 --- a/aes/src/x86.rs +++ /dev/null @@ -1,712 +0,0 @@ -pub(crate) mod ni; -#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] -pub(crate) mod vaes256; -#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] -pub(crate) mod vaes512; - -#[cfg(target_arch = "x86")] -use core::arch::x86 as arch; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64 as arch; - -use self::arch::*; -use crate::Block; -#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] -use cipher::consts::U64; -use cipher::{ - AlgorithmName, BlockCipherDecBackend, BlockCipherDecClosure, BlockCipherDecrypt, - BlockCipherEncBackend, BlockCipherEncClosure, BlockCipherEncrypt, BlockSizeUser, InOut, Key, - KeyInit, KeySizeUser, ParBlocksSizeUser, - consts::{U8, U16, U24, U32}, -}; -#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] -use cipher::{Array, InOutBuf, consts::U30, typenum::Unsigned}; -#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] -use core::cell::OnceCell; -use core::fmt; - -#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] -pub(crate) type Block30 = Array; -#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] -pub(crate) type Block64 = Array; - -pub(crate) mod features { - cpufeatures::new!(features_aes, "aes"); - cpufeatures::new!(features_avx, "avx"); - cpufeatures::new!(features_avx512f, "avx512f"); - cpufeatures::new!(features_vaes, "vaes"); - pub(crate) mod aes { - pub use super::features_aes::*; - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - pub(crate) mod avx { - pub use super::features_avx::*; - } - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - pub(crate) mod avx512f { - pub use super::features_avx512f::*; - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - pub(crate) mod vaes { - pub use super::features_vaes::*; - } -} - -type Simd128RoundKeys = [__m128i; ROUNDS]; -#[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] -type Simd256RoundKeys = [__m256i; ROUNDS]; -#[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] -type Simd512RoundKeys = [__m512i; ROUNDS]; - -#[derive(Clone)] -enum Backend { - Ni, - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - Vaes256, - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - Vaes512, -} - -#[derive(Clone, Copy)] -struct Features { - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - avx: self::features::avx::InitToken, - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - avx512f: self::features::avx512f::InitToken, - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - vaes: self::features::vaes::InitToken, -} - -impl Features { - fn new() -> Self { - Self { - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - avx: self::features::avx::init(), - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - avx512f: self::features::avx512f::init(), - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - vaes: self::features::vaes::init(), - } - } - - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - fn has_vaes256(&self) -> bool { - #[cfg(target_arch = "x86_64")] - if cfg!(aes_backend = "avx256") && self.vaes.get() && self.avx.get() { - return true; - } - false - } - - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - fn has_vaes512(&self) -> bool { - #[cfg(target_arch = "x86_64")] - if cfg!(aes_backend = "avx512") && self.vaes.get() && self.avx512f.get() { - return true; - } - false - } - - fn dispatch(&self) -> Backend { - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - if self.has_vaes512() { - return self::Backend::Vaes512; - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - if self.has_vaes256() { - return self::Backend::Vaes256; - } - Backend::Ni - } -} - -macro_rules! define_aes_impl { - ( - $name:tt, - $name_enc:ident, - $name_dec:ident, - $name_backend:ident, - $module:tt, - $key_size:ty, - $rounds:tt, - $doc:expr $(,)? - ) => { - mod $name_backend { - use super::*; - - #[derive(Clone)] - pub(crate) struct Ni<'a> { - pub(crate) keys: &'a Simd128RoundKeys<$rounds>, - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> Ni<'a> { - pub const fn par_blocks(&self) -> usize { - ::ParBlocksSize::USIZE - } - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> From<&Vaes256<'a>> for Ni<'a> { - fn from(backend: &Vaes256<'a>) -> Self { - Self { keys: backend.keys } - } - } - - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - #[derive(Clone)] - pub(crate) struct Vaes256<'a> { - #[allow(unused)] // TODO: remove once cfg flags are removed - pub(crate) features: Features, - pub(crate) keys: &'a Simd128RoundKeys<$rounds>, - pub(crate) simd_256_keys: OnceCell>, - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> Vaes256<'a> { - #[allow(unused)] // TODO: remove once cfg flags are removed - pub const fn par_blocks(&self) -> usize { - ::ParBlocksSize::USIZE - } - } - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - impl<'a> From<&Vaes512<'a>> for Vaes256<'a> { - fn from(backend: &Vaes512<'a>) -> Self { - Self { - features: backend.features, - keys: backend.keys, - simd_256_keys: OnceCell::new(), - } - } - } - - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - pub(crate) struct Vaes512<'a> { - pub(crate) features: Features, - pub(crate) keys: &'a Simd128RoundKeys<$rounds>, - pub(crate) simd_512_keys: OnceCell>, - } - } - - #[doc=$doc] - #[doc = "block cipher"] - #[derive(Clone)] - pub struct $name { - encrypt: $name_enc, - decrypt: $name_dec, - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name {} - - impl KeySizeUser for $name { - type KeySize = $key_size; - } - - impl KeyInit for $name { - #[inline] - fn new(key: &Key) -> Self { - let encrypt = $name_enc::new(key); - let decrypt = $name_dec::from(&encrypt); - Self { encrypt, decrypt } - } - } - - impl From<$name_enc> for $name { - #[inline] - fn from(encrypt: $name_enc) -> $name { - let decrypt = (&encrypt).into(); - Self { encrypt, decrypt } - } - } - - impl From<&$name_enc> for $name { - #[inline] - fn from(encrypt: &$name_enc) -> $name { - let decrypt = encrypt.into(); - let encrypt = encrypt.clone(); - Self { encrypt, decrypt } - } - } - - impl BlockSizeUser for $name { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name { - #[inline] - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - self.encrypt.encrypt_with_backend(f) - } - } - - impl BlockCipherDecrypt for $name { - #[inline] - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - self.decrypt.decrypt_with_backend(f) - } - } - - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name), " { .. }")) - } - } - - impl AlgorithmName for $name { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name)) - } - } - - #[doc=$doc] - #[doc = "block cipher (encrypt-only)"] - #[derive(Clone)] - pub struct $name_enc { - keys: Simd128RoundKeys<$rounds>, - features: Features, - } - - impl Drop for $name_enc { - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(&mut self.keys) - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_enc {} - - impl KeySizeUser for $name_enc { - type KeySize = $key_size; - } - - impl KeyInit for $name_enc { - #[inline] - fn new(key: &Key) -> Self { - // SAFETY: we enforce that this code is called only when - // target features required by `expand` were properly checked. - Self { - keys: unsafe { self::ni::expand::$module::expand_key(key.as_ref()) }, - features: Features::new(), - } - } - } - - impl BlockSizeUser for $name_enc { - type BlockSize = U16; - } - - impl BlockCipherEncrypt for $name_enc { - #[inline] - fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure) { - let features = self.features; - let keys = &self.keys; - match features.dispatch() { - self::Backend::Ni => f.call(&mut $name_backend::Ni { keys }), - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - self::Backend::Vaes256 => f.call(&mut $name_backend::Vaes256 { - features, - keys, - simd_256_keys: OnceCell::new(), - }), - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - self::Backend::Vaes512 => f.call(&mut $name_backend::Vaes512 { - features, - keys, - simd_512_keys: OnceCell::new(), - }), - } - } - } - - impl fmt::Debug for $name_enc { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_enc), " { .. }")) - } - } - - impl AlgorithmName for $name_enc { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_enc)) - } - } - - #[doc=$doc] - #[doc = "block cipher (decrypt-only)"] - #[derive(Clone)] - pub struct $name_dec { - keys: Simd128RoundKeys<$rounds>, - features: Features, - } - - impl Drop for $name_dec { - fn drop(&mut self) { - #[cfg(feature = "zeroize")] - unsafe { - zeroize::zeroize_flat_type(&mut self.keys) - } - } - } - - #[cfg(feature = "zeroize")] - impl zeroize::ZeroizeOnDrop for $name_dec {} - - impl KeySizeUser for $name_dec { - type KeySize = $key_size; - } - - impl KeyInit for $name_dec { - #[inline] - fn new(key: &Key) -> Self { - $name_enc::new(key).into() - } - } - - impl From<$name_enc> for $name_dec { - #[inline] - fn from(enc: $name_enc) -> $name_dec { - Self::from(&enc) - } - } - - impl From<&$name_enc> for $name_dec { - #[inline] - fn from(enc: &$name_enc) -> $name_dec { - Self { - keys: unsafe { self::ni::expand::inv_keys(&enc.keys) }, - features: enc.features.clone(), - } - } - } - - impl BlockSizeUser for $name_dec { - type BlockSize = U16; - } - - impl BlockCipherDecrypt for $name_dec { - #[inline] - fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure) { - let features = self.features; - let keys = &self.keys; - match features.dispatch() { - self::Backend::Ni => f.call(&mut $name_backend::Ni { keys }), - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - self::Backend::Vaes256 => f.call(&mut $name_backend::Vaes256 { - features, - keys, - simd_256_keys: OnceCell::new(), - }), - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - self::Backend::Vaes512 => f.call(&mut $name_backend::Vaes512 { - features, - keys, - simd_512_keys: OnceCell::new(), - }), - } - } - } - - impl fmt::Debug for $name_dec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.write_str(concat!(stringify!($name_dec), " { .. }")) - } - } - - impl AlgorithmName for $name_dec { - fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(stringify!($name_dec)) - } - } - - impl<'a> BlockSizeUser for $name_backend::Ni<'a> { - type BlockSize = U16; - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> BlockSizeUser for $name_backend::Vaes256<'a> { - type BlockSize = U16; - } - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - impl<'a> BlockSizeUser for $name_backend::Vaes512<'a> { - type BlockSize = U16; - } - - impl<'a> ParBlocksSizeUser for $name_backend::Ni<'a> { - type ParBlocksSize = U8; - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> ParBlocksSizeUser for $name_backend::Vaes256<'a> { - // Block size of 30 is chosen based on AVX2's 16 YMM registers. - // - // * 1 register holds 2 keys per round (loads interleaved with rounds) - // * 15 registers hold 2 data blocks - // - // This gives (16 - 1 ) * 2 = 30 . - type ParBlocksSize = U30; - } - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - impl<'a> ParBlocksSizeUser for $name_backend::Vaes512<'a> { - // Block size of 64 is chosen based on AVX512's 32 ZMM registers. - // - // * 11, 13, 15 registers for keys, correspond to AES-128, AES-192, AES-256 - // * 11, 13, 15 registers hold 4 keys each (no interleaved loading like VAES256) - // * 16 registers hold 4 data blocks - // * 1-4 registers remain unused (could use them but probably not worth it) - // - // This gives (32 - 15 - 1 ) * 4 = 64 . - type ParBlocksSize = U64; - } - - impl<'a> BlockCipherEncBackend for $name_backend::Ni<'a> { - #[inline] - fn encrypt_block(&self, block: InOut<'_, '_, Block>) { - unsafe { - self::ni::encdec::encrypt(self.keys, block); - } - } - #[inline] - fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks>) { - unsafe { - self::ni::encdec::encrypt_par(self.keys, blocks); - } - } - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> BlockCipherEncBackend for $name_backend::Vaes256<'a> { - #[inline] - fn encrypt_block(&self, block: InOut<'_, '_, Block>) { - unsafe { - self::ni::encdec::encrypt(self.keys, block); - } - } - #[inline] - fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks>) { - unsafe { - let simd_256_keys = self - .simd_256_keys - .get_or_init(|| vaes256::encdec::broadcast_keys(&self.keys)); - vaes256::encdec::encrypt30(simd_256_keys, blocks); - } - } - #[inline] - fn encrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) { - let backend = self; - - let mut rem = blocks.len(); - let (mut iptr, mut optr) = blocks.into_raw(); - - let backend = $name_backend::Ni::from(backend); - while rem >= backend.par_blocks() { - let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) }; - backend.encrypt_par_blocks(blocks); - rem -= backend.par_blocks(); - iptr = unsafe { iptr.add(backend.par_blocks()) }; - optr = unsafe { optr.add(backend.par_blocks()) }; - } - - while rem > 0 { - let block = unsafe { InOut::from_raw(iptr, optr) }; - backend.encrypt_block(block); - rem -= 1; - iptr = unsafe { iptr.add(1) }; - optr = unsafe { optr.add(1) }; - } - } - } - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - impl<'a> BlockCipherEncBackend for $name_backend::Vaes512<'a> { - #[inline] - fn encrypt_block(&self, block: InOut<'_, '_, Block>) { - unsafe { - self::ni::encdec::encrypt(self.keys, block); - } - } - #[inline] - fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks>) { - unsafe { - let simd_512_keys = self - .simd_512_keys - .get_or_init(|| vaes512::encdec::broadcast_keys(&self.keys)); - vaes512::encdec::encrypt64(simd_512_keys, blocks); - } - } - #[inline] - fn encrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) { - let backend = self; - - let mut rem = blocks.len(); - let (mut iptr, mut optr) = blocks.into_raw(); - - let backend = &$name_backend::Vaes256::from(backend); - if backend.features.has_vaes256() { - while rem >= backend.par_blocks() { - let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) }; - backend.encrypt_par_blocks(blocks); - rem -= backend.par_blocks(); - iptr = unsafe { iptr.add(backend.par_blocks()) }; - optr = unsafe { optr.add(backend.par_blocks()) }; - } - } - - let backend = &$name_backend::Ni::from(backend); - while rem >= backend.par_blocks() { - let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) }; - backend.encrypt_par_blocks(blocks); - rem -= backend.par_blocks(); - iptr = unsafe { iptr.add(backend.par_blocks()) }; - optr = unsafe { optr.add(backend.par_blocks()) }; - } - - while rem > 0 { - let block = unsafe { InOut::from_raw(iptr, optr) }; - backend.encrypt_block(block); - rem -= 1; - iptr = unsafe { iptr.add(1) }; - optr = unsafe { optr.add(1) }; - } - } - } - - impl<'a> BlockCipherDecBackend for $name_backend::Ni<'a> { - #[inline] - fn decrypt_block(&self, block: InOut<'_, '_, Block>) { - unsafe { - self::ni::encdec::decrypt(self.keys, block); - } - } - #[inline] - fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks>) { - unsafe { - self::ni::encdec::decrypt_par(self.keys, blocks); - } - } - } - #[cfg(all(target_arch = "x86_64", any(aes_backend = "avx256", aes_backend = "avx512")))] - impl<'a> BlockCipherDecBackend for $name_backend::Vaes256<'a> { - #[inline] - fn decrypt_block(&self, block: InOut<'_, '_, Block>) { - unsafe { - self::ni::encdec::decrypt(self.keys, block); - } - } - #[inline] - fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks>) { - unsafe { - let simd_256_keys = self - .simd_256_keys - .get_or_init(|| vaes256::encdec::broadcast_keys(&self.keys)); - vaes256::encdec::decrypt30(simd_256_keys, blocks); - } - } - #[inline] - fn decrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) { - let backend = self; - - let mut rem = blocks.len(); - let (mut iptr, mut optr) = blocks.into_raw(); - - let backend = $name_backend::Ni::from(backend); - while rem >= backend.par_blocks() { - let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) }; - backend.decrypt_par_blocks(blocks); - rem -= backend.par_blocks(); - iptr = unsafe { iptr.add(backend.par_blocks()) }; - optr = unsafe { optr.add(backend.par_blocks()) }; - } - - while rem > 0 { - let block = unsafe { InOut::from_raw(iptr, optr) }; - backend.decrypt_block(block); - rem -= 1; - iptr = unsafe { iptr.add(1) }; - optr = unsafe { optr.add(1) }; - } - } - } - #[cfg(all(target_arch = "x86_64", aes_backend = "avx512"))] - impl<'a> BlockCipherDecBackend for $name_backend::Vaes512<'a> { - #[inline] - fn decrypt_block(&self, block: InOut<'_, '_, Block>) { - unsafe { - self::ni::encdec::decrypt(self.keys, block); - } - } - #[inline] - fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, cipher::ParBlocks>) { - unsafe { - let simd_512_keys = self - .simd_512_keys - .get_or_init(|| vaes512::encdec::broadcast_keys(&self.keys)); - vaes512::encdec::decrypt64(simd_512_keys, blocks); - } - } - #[inline] - fn decrypt_tail_blocks(&self, blocks: InOutBuf<'_, '_, Block>) { - let backend = self; - - let mut rem = blocks.len(); - let (mut iptr, mut optr) = blocks.into_raw(); - - let backend = &$name_backend::Vaes256::from(backend); - if backend.features.has_vaes256() { - while rem >= backend.par_blocks() { - let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) }; - backend.decrypt_par_blocks(blocks); - rem -= backend.par_blocks(); - iptr = unsafe { iptr.add(backend.par_blocks()) }; - optr = unsafe { optr.add(backend.par_blocks()) }; - } - } - - let backend = &$name_backend::Ni::from(backend); - while rem >= backend.par_blocks() { - let blocks = unsafe { InOut::from_raw(iptr.cast(), optr.cast()) }; - backend.decrypt_par_blocks(blocks); - rem -= backend.par_blocks(); - iptr = unsafe { iptr.add(backend.par_blocks()) }; - optr = unsafe { optr.add(backend.par_blocks()) }; - } - - while rem > 0 { - let block = unsafe { InOut::from_raw(iptr, optr) }; - backend.decrypt_block(block); - rem -= 1; - iptr = unsafe { iptr.add(1) }; - optr = unsafe { optr.add(1) }; - } - } - } - }; -} - -define_aes_impl!( - Aes128, - Aes128Enc, - Aes128Dec, - aes128_backend, - aes128, - U16, - 11, - "AES-128", -); - -define_aes_impl!( - Aes192, - Aes192Enc, - Aes192Dec, - aes192_backend, - aes192, - U24, - 13, - "AES-192", -); - -define_aes_impl!( - Aes256, - Aes256Enc, - Aes256Dec, - aes256_backend, - aes256, - U32, - 15, - "AES-256", -); diff --git a/aes/src/x86/ni.rs b/aes/src/x86/ni.rs deleted file mode 100644 index c35b1b86b..000000000 --- a/aes/src/x86/ni.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! AES block ciphers implementation using AES-NI instruction set. -//! -//! Ciphers functionality is accessed using `BlockCipher` trait from the -//! [`cipher`](https://docs.rs/cipher) crate. -//! -//! # Vulnerability -//! Lazy FP state restory vulnerability can allow local process to leak content -//! of the FPU register, in which round keys are stored. This vulnerability -//! can be mitigated at the operating system level by installing relevant -//! patches. (i.e. keep your OS updated!) More info: -//! - [Intel advisory](https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00145.html) -//! - [Wikipedia](https://en.wikipedia.org/wiki/Lazy_FP_state_restore) -//! -//! # Related documents -//! - [Intel AES-NI whitepaper](https://software.intel.com/sites/default/files/article/165683/aes-wp-2012-09-22-v01.pdf) -//! - [Use of the AES Instruction Set](https://www.cosic.esat.kuleuven.be/ecrypt/AESday/slides/Use_of_the_AES_Instruction_Set.pdf) - -pub(super) mod encdec; -pub(super) mod expand; -#[cfg(test)] -mod test_expand; - -#[cfg(feature = "hazmat")] -pub(crate) mod hazmat; diff --git a/aes/src/x86/ni/encdec.rs b/aes/src/x86/ni/encdec.rs deleted file mode 100644 index b68ed5f9a..000000000 --- a/aes/src/x86/ni/encdec.rs +++ /dev/null @@ -1,161 +0,0 @@ -#![allow(unsafe_op_in_unsafe_fn)] - -use crate::Block; -use crate::x86::arch::*; -use cipher::{ - array::{Array, ArraySize}, - inout::InOut, -}; - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn encrypt( - keys: &[__m128i; KEYS], - block: InOut<'_, '_, Block>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (block_in, block_out) = block.into_raw(); - let mut b = _mm_loadu_si128(block_in.cast()); - b = _mm_xor_si128(b, keys[0]); - for &key in &keys[1..KEYS - 1] { - b = _mm_aesenc_si128(b, key); - } - b = _mm_aesenclast_si128(b, keys[KEYS - 1]); - _mm_storeu_si128(block_out.cast(), b); -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn decrypt( - keys: &[__m128i; KEYS], - block: InOut<'_, '_, Block>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (block_in, block_out) = block.into_raw(); - let mut b = _mm_loadu_si128(block_in.cast()); - b = _mm_xor_si128(b, keys[0]); - for &key in &keys[1..KEYS - 1] { - b = _mm_aesdec_si128(b, key); - } - b = _mm_aesdeclast_si128(b, keys[KEYS - 1]); - _mm_storeu_si128(block_out.cast(), b); -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn encrypt_par( - keys: &[__m128i; KEYS], - blocks: InOut<'_, '_, Array>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (blocks_in, blocks_out) = blocks.into_raw(); - let mut b = load(blocks_in); - - // Loop over keys is intentionally not used here to force inlining - xor(&mut b, keys[0]); - aesenc(&mut b, keys[1]); - aesenc(&mut b, keys[2]); - aesenc(&mut b, keys[3]); - aesenc(&mut b, keys[4]); - aesenc(&mut b, keys[5]); - aesenc(&mut b, keys[6]); - aesenc(&mut b, keys[7]); - aesenc(&mut b, keys[8]); - aesenc(&mut b, keys[9]); - if KEYS >= 13 { - aesenc(&mut b, keys[10]); - aesenc(&mut b, keys[11]); - } - if KEYS == 15 { - aesenc(&mut b, keys[12]); - aesenc(&mut b, keys[13]); - } - aesenclast(&mut b, keys[KEYS - 1]); - store(blocks_out, b); -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn decrypt_par( - keys: &[__m128i; KEYS], - blocks: InOut<'_, '_, Array>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (blocks_in, blocks_out) = blocks.into_raw(); - let mut b = load(blocks_in); - - // Loop over keys is intentionally not used here to force inlining - xor(&mut b, keys[0]); - aesdec(&mut b, keys[1]); - aesdec(&mut b, keys[2]); - aesdec(&mut b, keys[3]); - aesdec(&mut b, keys[4]); - aesdec(&mut b, keys[5]); - aesdec(&mut b, keys[6]); - aesdec(&mut b, keys[7]); - aesdec(&mut b, keys[8]); - aesdec(&mut b, keys[9]); - if KEYS >= 13 { - aesdec(&mut b, keys[10]); - aesdec(&mut b, keys[11]); - } - if KEYS == 15 { - aesdec(&mut b, keys[12]); - aesdec(&mut b, keys[13]); - } - aesdeclast(&mut b, keys[KEYS - 1]); - store(blocks_out, b); -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn load(blocks: *const Array) -> Array<__m128i, N> { - let p = blocks.cast::<__m128i>(); - let mut res: Array<__m128i, N> = core::mem::zeroed(); - for i in 0..N::USIZE { - res[i] = _mm_loadu_si128(p.add(i)); - } - res -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn store(blocks: *mut Array, b: Array<__m128i, N>) { - let p = blocks.cast::<__m128i>(); - for i in 0..N::USIZE { - _mm_storeu_si128(p.add(i), b[i]); - } -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn xor(blocks: &mut Array<__m128i, N>, key: __m128i) { - for block in blocks { - *block = _mm_xor_si128(*block, key); - } -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn aesenc(blocks: &mut Array<__m128i, N>, key: __m128i) { - for block in blocks { - *block = _mm_aesenc_si128(*block, key); - } -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn aesenclast(blocks: &mut Array<__m128i, N>, key: __m128i) { - for block in blocks { - *block = _mm_aesenclast_si128(*block, key); - } -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn aesdec(blocks: &mut Array<__m128i, N>, key: __m128i) { - for block in blocks { - *block = _mm_aesdec_si128(*block, key); - } -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn aesdeclast(blocks: &mut Array<__m128i, N>, key: __m128i) { - for block in blocks { - *block = _mm_aesdeclast_si128(*block, key); - } -} diff --git a/aes/src/x86/ni/expand.rs b/aes/src/x86/ni/expand.rs deleted file mode 100644 index 68bdb357f..000000000 --- a/aes/src/x86/ni/expand.rs +++ /dev/null @@ -1,219 +0,0 @@ -#![allow(unsafe_op_in_unsafe_fn)] - -use crate::x86::arch::*; -use core::mem::{transmute, zeroed}; - -pub(super) type Aes128RoundKeys = [__m128i; 11]; -pub(super) type Aes192RoundKeys = [__m128i; 13]; -pub(super) type Aes256RoundKeys = [__m128i; 15]; - -pub(crate) mod aes128 { - use super::*; - - #[target_feature(enable = "aes")] - pub(crate) unsafe fn expand_key(key: &[u8; 16]) -> Aes128RoundKeys { - unsafe fn expand_round(keys: &mut Aes128RoundKeys, pos: usize) { - let mut t1 = keys[pos - 1]; - let mut t2; - let mut t3; - - t2 = _mm_aeskeygenassist_si128(t1, RK); - t2 = _mm_shuffle_epi32(t2, 0xff); - t3 = _mm_slli_si128(t1, 0x4); - t1 = _mm_xor_si128(t1, t3); - t3 = _mm_slli_si128(t3, 0x4); - t1 = _mm_xor_si128(t1, t3); - t3 = _mm_slli_si128(t3, 0x4); - t1 = _mm_xor_si128(t1, t3); - t1 = _mm_xor_si128(t1, t2); - - keys[pos] = t1; - } - - let mut keys: Aes128RoundKeys = zeroed(); - let k = _mm_loadu_si128(key.as_ptr().cast()); - keys[0] = k; - - let kr = &mut keys; - expand_round::<0x01>(kr, 1); - expand_round::<0x02>(kr, 2); - expand_round::<0x04>(kr, 3); - expand_round::<0x08>(kr, 4); - expand_round::<0x10>(kr, 5); - expand_round::<0x20>(kr, 6); - expand_round::<0x40>(kr, 7); - expand_round::<0x80>(kr, 8); - expand_round::<0x1B>(kr, 9); - expand_round::<0x36>(kr, 10); - - keys - } -} - -pub(crate) mod aes192 { - use super::*; - - #[target_feature(enable = "aes")] - pub(crate) unsafe fn expand_key(key: &[u8; 24]) -> Aes192RoundKeys { - unsafe fn shuffle(a: __m128i, b: __m128i, i: usize) -> __m128i { - let a: [u64; 2] = transmute(a); - let b: [u64; 2] = transmute(b); - transmute([a[i], b[0]]) - } - - #[target_feature(enable = "aes")] - unsafe fn expand_round( - mut t1: __m128i, - mut t3: __m128i, - ) -> (__m128i, __m128i) { - let (mut t2, mut t4); - - t2 = _mm_aeskeygenassist_si128(t3, RK); - t2 = _mm_shuffle_epi32(t2, 0x55); - t4 = _mm_slli_si128(t1, 0x4); - t1 = _mm_xor_si128(t1, t4); - t4 = _mm_slli_si128(t4, 0x4); - t1 = _mm_xor_si128(t1, t4); - t4 = _mm_slli_si128(t4, 0x4); - t1 = _mm_xor_si128(t1, t4); - t1 = _mm_xor_si128(t1, t2); - t2 = _mm_shuffle_epi32(t1, 0xff); - t4 = _mm_slli_si128(t3, 0x4); - t3 = _mm_xor_si128(t3, t4); - t3 = _mm_xor_si128(t3, t2); - - (t1, t3) - } - - let mut keys: Aes192RoundKeys = zeroed(); - // We are being extra pedantic here to remove out-of-bound access. - // This should be optimized into movups, movsd sequence. - let (k0, k1l) = { - let mut t = [0u8; 32]; - t[..key.len()].copy_from_slice(key); - ( - _mm_loadu_si128(t.as_ptr().cast()), - _mm_loadu_si128(t.as_ptr().offset(16).cast()), - ) - }; - - keys[0] = k0; - - let (k1_2, k2r) = expand_round::<0x01>(k0, k1l); - keys[1] = shuffle(k1l, k1_2, 0); - keys[2] = shuffle(k1_2, k2r, 1); - - let (k3, k4l) = expand_round::<0x02>(k1_2, k2r); - keys[3] = k3; - - let (k4_5, k5r) = expand_round::<0x04>(k3, k4l); - let k4 = shuffle(k4l, k4_5, 0); - let k5 = shuffle(k4_5, k5r, 1); - keys[4] = k4; - keys[5] = k5; - - let (k6, k7l) = expand_round::<0x08>(k4_5, k5r); - keys[6] = k6; - - let (k7_8, k8r) = expand_round::<0x10>(k6, k7l); - keys[7] = shuffle(k7l, k7_8, 0); - keys[8] = shuffle(k7_8, k8r, 1); - - let (k9, k10l) = expand_round::<0x20>(k7_8, k8r); - keys[9] = k9; - - let (k10_11, k11r) = expand_round::<0x40>(k9, k10l); - keys[10] = shuffle(k10l, k10_11, 0); - keys[11] = shuffle(k10_11, k11r, 1); - - let (k12, _) = expand_round::<0x80>(k10_11, k11r); - keys[12] = k12; - - keys - } -} - -pub(crate) mod aes256 { - use super::*; - - #[target_feature(enable = "aes")] - pub(crate) unsafe fn expand_key(key: &[u8; 32]) -> Aes256RoundKeys { - unsafe fn expand_round(keys: &mut Aes256RoundKeys, pos: usize) { - let mut t1 = keys[pos - 2]; - let mut t2; - let mut t3 = keys[pos - 1]; - let mut t4; - - t2 = _mm_aeskeygenassist_si128(t3, RK); - t2 = _mm_shuffle_epi32(t2, 0xff); - t4 = _mm_slli_si128(t1, 0x4); - t1 = _mm_xor_si128(t1, t4); - t4 = _mm_slli_si128(t4, 0x4); - t1 = _mm_xor_si128(t1, t4); - t4 = _mm_slli_si128(t4, 0x4); - t1 = _mm_xor_si128(t1, t4); - t1 = _mm_xor_si128(t1, t2); - - keys[pos] = t1; - - t4 = _mm_aeskeygenassist_si128(t1, 0x00); - t2 = _mm_shuffle_epi32(t4, 0xaa); - t4 = _mm_slli_si128(t3, 0x4); - t3 = _mm_xor_si128(t3, t4); - t4 = _mm_slli_si128(t4, 0x4); - t3 = _mm_xor_si128(t3, t4); - t4 = _mm_slli_si128(t4, 0x4); - t3 = _mm_xor_si128(t3, t4); - t3 = _mm_xor_si128(t3, t2); - - keys[pos + 1] = t3; - } - - unsafe fn expand_round_last(keys: &mut Aes256RoundKeys, pos: usize) { - let mut t1 = keys[pos - 2]; - let mut t2; - let t3 = keys[pos - 1]; - let mut t4; - - t2 = _mm_aeskeygenassist_si128(t3, RK); - t2 = _mm_shuffle_epi32(t2, 0xff); - t4 = _mm_slli_si128(t1, 0x4); - t1 = _mm_xor_si128(t1, t4); - t4 = _mm_slli_si128(t4, 0x4); - t1 = _mm_xor_si128(t1, t4); - t4 = _mm_slli_si128(t4, 0x4); - t1 = _mm_xor_si128(t1, t4); - t1 = _mm_xor_si128(t1, t2); - - keys[pos] = t1; - } - - let mut keys: Aes256RoundKeys = zeroed(); - - let kp = key.as_ptr().cast::<__m128i>(); - keys[0] = _mm_loadu_si128(kp); - keys[1] = _mm_loadu_si128(kp.add(1)); - - let k = &mut keys; - expand_round::<0x01>(k, 2); - expand_round::<0x02>(k, 4); - expand_round::<0x04>(k, 6); - expand_round::<0x08>(k, 8); - expand_round::<0x10>(k, 10); - expand_round::<0x20>(k, 12); - expand_round_last::<0x40>(k, 14); - - keys - } -} - -#[target_feature(enable = "aes")] -pub(crate) unsafe fn inv_keys(keys: &[__m128i; N]) -> [__m128i; N] { - let mut inv_keys: [__m128i; N] = zeroed(); - inv_keys[0] = keys[N - 1]; - for i in 1..N - 1 { - inv_keys[i] = _mm_aesimc_si128(keys[N - 1 - i]); - } - inv_keys[N - 1] = keys[0]; - inv_keys -} diff --git a/aes/src/x86/ni/test_expand.rs b/aes/src/x86/ni/test_expand.rs deleted file mode 100644 index 6524ef74a..000000000 --- a/aes/src/x86/ni/test_expand.rs +++ /dev/null @@ -1,291 +0,0 @@ -use crate::x86::arch::*; -use crate::x86::ni::expand::*; -use hex_literal::hex; - -pub(crate) fn check(a: &[__m128i], b: &[[u64; 2]]) { - assert_eq!(a.len(), b.len()); - for (v1, v2) in a.iter().zip(b) { - let t1: [u64; 2] = unsafe { core::mem::transmute(*v1) }; - let t2 = [v2[0].to_be(), v2[1].to_be()]; - assert_eq!(t1, t2); - } -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes128_expand_key_test() { - let keys = [0x00; 16]; - check( - &unsafe { aes128::expand_key(&keys) }, - &[ - [0x0000000000000000, 0x0000000000000000], - [0x6263636362636363, 0x6263636362636363], - [0x9b9898c9f9fbfbaa, 0x9b9898c9f9fbfbaa], - [0x90973450696ccffa, 0xf2f457330b0fac99], - [0xee06da7b876a1581, 0x759e42b27e91ee2b], - [0x7f2e2b88f8443e09, 0x8dda7cbbf34b9290], - [0xec614b851425758c, 0x99ff09376ab49ba7], - [0x217517873550620b, 0xacaf6b3cc61bf09b], - [0x0ef903333ba96138, 0x97060a04511dfa9f], - [0xb1d4d8e28a7db9da, 0x1d7bb3de4c664941], - [0xb4ef5bcb3e92e211, 0x23e951cf6f8f188e], - ], - ); - - let keys = [0xff; 16]; - check( - &unsafe { aes128::expand_key(&keys) }, - &[ - [0xffffffffffffffff, 0xffffffffffffffff], - [0xe8e9e9e917161616, 0xe8e9e9e917161616], - [0xadaeae19bab8b80f, 0x525151e6454747f0], - [0x090e2277b3b69a78, 0xe1e7cb9ea4a08c6e], - [0xe16abd3e52dc2746, 0xb33becd8179b60b6], - [0xe5baf3ceb766d488, 0x045d385013c658e6], - [0x71d07db3c6b6a93b, 0xc2eb916bd12dc98d], - [0xe90d208d2fbb89b6, 0xed5018dd3c7dd150], - [0x96337366b988fad0, 0x54d8e20d68a5335d], - [0x8bf03f233278c5f3, 0x66a027fe0e0514a3], - [0xd60a3588e472f07b, 0x82d2d7858cd7c326], - ], - ); - - let keys = hex!("000102030405060708090a0b0c0d0e0f"); - check( - &unsafe { aes128::expand_key(&keys) }, - &[ - [0x0001020304050607, 0x08090a0b0c0d0e0f], - [0xd6aa74fdd2af72fa, 0xdaa678f1d6ab76fe], - [0xb692cf0b643dbdf1, 0xbe9bc5006830b3fe], - [0xb6ff744ed2c2c9bf, 0x6c590cbf0469bf41], - [0x47f7f7bc95353e03, 0xf96c32bcfd058dfd], - [0x3caaa3e8a99f9deb, 0x50f3af57adf622aa], - [0x5e390f7df7a69296, 0xa7553dc10aa31f6b], - [0x14f9701ae35fe28c, 0x440adf4d4ea9c026], - [0x47438735a41c65b9, 0xe016baf4aebf7ad2], - [0x549932d1f0855768, 0x1093ed9cbe2c974e], - [0x13111d7fe3944a17, 0xf307a78b4d2b30c5], - ], - ); - - let keys = hex!("6920e299a5202a6d656e636869746f2a"); - check( - &unsafe { aes128::expand_key(&keys) }, - &[ - [0x6920e299a5202a6d, 0x656e636869746f2a], - [0xfa8807605fa82d0d, 0x3ac64e6553b2214f], - [0xcf75838d90ddae80, 0xaa1be0e5f9a9c1aa], - [0x180d2f1488d08194, 0x22cb6171db62a0db], - [0xbaed96ad323d1739, 0x10f67648cb94d693], - [0x881b4ab2ba265d8b, 0xaad02bc36144fd50], - [0xb34f195d096944d6, 0xa3b96f15c2fd9245], - [0xa7007778ae6933ae, 0x0dd05cbbcf2dcefe], - [0xff8bccf251e2ff5c, 0x5c32a3e7931f6d19], - [0x24b7182e7555e772, 0x29674495ba78298c], - [0xae127cdadb479ba8, 0xf220df3d4858f6b1], - ], - ); - - let keys = hex!("2b7e151628aed2a6abf7158809cf4f3c"); - check( - &unsafe { aes128::expand_key(&keys) }, - &[ - [0x2b7e151628aed2a6, 0xabf7158809cf4f3c], - [0xa0fafe1788542cb1, 0x23a339392a6c7605], - [0xf2c295f27a96b943, 0x5935807a7359f67f], - [0x3d80477d4716fe3e, 0x1e237e446d7a883b], - [0xef44a541a8525b7f, 0xb671253bdb0bad00], - [0xd4d1c6f87c839d87, 0xcaf2b8bc11f915bc], - [0x6d88a37a110b3efd, 0xdbf98641ca0093fd], - [0x4e54f70e5f5fc9f3, 0x84a64fb24ea6dc4f], - [0xead27321b58dbad2, 0x312bf5607f8d292f], - [0xac7766f319fadc21, 0x28d12941575c006e], - [0xd014f9a8c9ee2589, 0xe13f0cc8b6630ca6], - ], - ); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes192_expand_key_test() { - let keys = [0x00; 24]; - check( - &unsafe { aes192::expand_key(&keys) }, - &[ - [0x0000000000000000, 0x0000000000000000], - [0x0000000000000000, 0x6263636362636363], - [0x6263636362636363, 0x6263636362636363], - [0x9b9898c9f9fbfbaa, 0x9b9898c9f9fbfbaa], - [0x9b9898c9f9fbfbaa, 0x90973450696ccffa], - [0xf2f457330b0fac99, 0x90973450696ccffa], - [0xc81d19a9a171d653, 0x53858160588a2df9], - [0xc81d19a9a171d653, 0x7bebf49bda9a22c8], - [0x891fa3a8d1958e51, 0x198897f8b8f941ab], - [0xc26896f718f2b43f, 0x91ed1797407899c6], - [0x59f00e3ee1094f95, 0x83ecbc0f9b1e0830], - [0x0af31fa74a8b8661, 0x137b885ff272c7ca], - [0x432ac886d834c0b6, 0xd2c7df11984c5970], - ], - ); - - let keys = [0xff; 24]; - check( - &unsafe { aes192::expand_key(&keys) }, - &[ - [0xffffffffffffffff, 0xffffffffffffffff], - [0xffffffffffffffff, 0xe8e9e9e917161616], - [0xe8e9e9e917161616, 0xe8e9e9e917161616], - [0xadaeae19bab8b80f, 0x525151e6454747f0], - [0xadaeae19bab8b80f, 0xc5c2d8ed7f7a60e2], - [0x2d2b3104686c76f4, 0xc5c2d8ed7f7a60e2], - [0x1712403f686820dd, 0x454311d92d2f672d], - [0xe8edbfc09797df22, 0x8f8cd3b7e7e4f36a], - [0xa2a7e2b38f88859e, 0x67653a5ef0f2e57c], - [0x2655c33bc1b13051, 0x6316d2e2ec9e577c], - [0x8bfb6d227b09885e, 0x67919b1aa620ab4b], - [0xc53679a929a82ed5, 0xa25343f7d95acba9], - [0x598e482fffaee364, 0x3a989acd1330b418], - ], - ); - - let keys = hex!("000102030405060708090a0b0c0d0e0f1011121314151617"); - check( - &unsafe { aes192::expand_key(&keys) }, - &[ - [0x0001020304050607, 0x08090a0b0c0d0e0f], - [0x1011121314151617, 0x5846f2f95c43f4fe], - [0x544afef55847f0fa, 0x4856e2e95c43f4fe], - [0x40f949b31cbabd4d, 0x48f043b810b7b342], - [0x58e151ab04a2a555, 0x7effb5416245080c], - [0x2ab54bb43a02f8f6, 0x62e3a95d66410c08], - [0xf501857297448d7e, 0xbdf1c6ca87f33e3c], - [0xe510976183519b69, 0x34157c9ea351f1e0], - [0x1ea0372a99530916, 0x7c439e77ff12051e], - [0xdd7e0e887e2fff68, 0x608fc842f9dcc154], - [0x859f5f237a8d5a3d, 0xc0c02952beefd63a], - [0xde601e7827bcdf2c, 0xa223800fd8aeda32], - [0xa4970a331a78dc09, 0xc418c271e3a41d5d], - ], - ); - - let keys = hex!("8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"); - check( - &unsafe { aes192::expand_key(&keys) }, - &[ - [0x8e73b0f7da0e6452, 0xc810f32b809079e5], - [0x62f8ead2522c6b7b, 0xfe0c91f72402f5a5], - [0xec12068e6c827f6b, 0x0e7a95b95c56fec2], - [0x4db7b4bd69b54118, 0x85a74796e92538fd], - [0xe75fad44bb095386, 0x485af05721efb14f], - [0xa448f6d94d6dce24, 0xaa326360113b30e6], - [0xa25e7ed583b1cf9a, 0x27f939436a94f767], - [0xc0a69407d19da4e1, 0xec1786eb6fa64971], - [0x485f703222cb8755, 0xe26d135233f0b7b3], - [0x40beeb282f18a259, 0x6747d26b458c553e], - [0xa7e1466c9411f1df, 0x821f750aad07d753], - [0xca4005388fcc5006, 0x282d166abc3ce7b5], - [0xe98ba06f448c773c, 0x8ecc720401002202], - ], - ); -} - -#[test] -#[cfg_attr( - not(target_feature = "aes"), - ignore = "requires enabled `aes` target feature" -)] -fn aes256_expand_key_test() { - let keys = [0x00; 32]; - check( - &unsafe { aes256::expand_key(&keys) }, - &[ - [0x0000000000000000, 0x0000000000000000], - [0x0000000000000000, 0x0000000000000000], - [0x6263636362636363, 0x6263636362636363], - [0xaafbfbfbaafbfbfb, 0xaafbfbfbaafbfbfb], - [0x6f6c6ccf0d0f0fac, 0x6f6c6ccf0d0f0fac], - [0x7d8d8d6ad7767691, 0x7d8d8d6ad7767691], - [0x5354edc15e5be26d, 0x31378ea23c38810e], - [0x968a81c141fcf750, 0x3c717a3aeb070cab], - [0x9eaa8f28c0f16d45, 0xf1c6e3e7cdfe62e9], - [0x2b312bdf6acddc8f, 0x56bca6b5bdbbaa1e], - [0x6406fd52a4f79017, 0x553173f098cf1119], - [0x6dbba90b07767584, 0x51cad331ec71792f], - [0xe7b0e89c4347788b, 0x16760b7b8eb91a62], - [0x74ed0ba1739b7e25, 0x2251ad14ce20d43b], - [0x10f80a1753bf729c, 0x45c979e7cb706385], - ], - ); - - let keys = [0xff; 32]; - check( - &unsafe { aes256::expand_key(&keys) }, - &[ - [0xffffffffffffffff, 0xffffffffffffffff], - [0xffffffffffffffff, 0xffffffffffffffff], - [0xe8e9e9e917161616, 0xe8e9e9e917161616], - [0x0fb8b8b8f0474747, 0x0fb8b8b8f0474747], - [0x4a4949655d5f5f73, 0xb5b6b69aa2a0a08c], - [0x355858dcc51f1f9b, 0xcaa7a7233ae0e064], - [0xafa80ae5f2f75596, 0x4741e30ce5e14380], - [0xeca0421129bf5d8a, 0xe318faa9d9f81acd], - [0xe60ab7d014fde246, 0x53bc014ab65d42ca], - [0xa2ec6e658b5333ef, 0x684bc946b1b3d38b], - [0x9b6c8a188f91685e, 0xdc2d69146a702bde], - [0xa0bd9f782beeac97, 0x43a565d1f216b65a], - [0xfc22349173b35ccf, 0xaf9e35dbc5ee1e05], - [0x0695ed132d7b4184, 0x6ede24559cc8920f], - [0x546d424f27de1e80, 0x88402b5b4dae355e], - ], - ); - - let keys = hex!("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"); - check( - &unsafe { aes256::expand_key(&keys) }, - &[ - [0x0001020304050607, 0x08090a0b0c0d0e0f], - [0x1011121314151617, 0x18191a1b1c1d1e1f], - [0xa573c29fa176c498, 0xa97fce93a572c09c], - [0x1651a8cd0244beda, 0x1a5da4c10640bade], - [0xae87dff00ff11b68, 0xa68ed5fb03fc1567], - [0x6de1f1486fa54f92, 0x75f8eb5373b8518d], - [0xc656827fc9a79917, 0x6f294cec6cd5598b], - [0x3de23a75524775e7, 0x27bf9eb45407cf39], - [0x0bdc905fc27b0948, 0xad5245a4c1871c2f], - [0x45f5a66017b2d387, 0x300d4d33640a820a], - [0x7ccff71cbeb4fe54, 0x13e6bbf0d261a7df], - [0xf01afafee7a82979, 0xd7a5644ab3afe640], - [0x2541fe719bf50025, 0x8813bbd55a721c0a], - [0x4e5a6699a9f24fe0, 0x7e572baacdf8cdea], - [0x24fc79ccbf0979e9, 0x371ac23c6d68de36], - ], - ); - - let keys = hex!("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"); - check( - &unsafe { aes256::expand_key(&keys) }, - &[ - [0x603deb1015ca71be, 0x2b73aef0857d7781], - [0x1f352c073b6108d7, 0x2d9810a30914dff4], - [0x9ba354118e6925af, 0xa51a8b5f2067fcde], - [0xa8b09c1a93d194cd, 0xbe49846eb75d5b9a], - [0xd59aecb85bf3c917, 0xfee94248de8ebe96], - [0xb5a9328a2678a647, 0x983122292f6c79b3], - [0x812c81addadf48ba, 0x24360af2fab8b464], - [0x98c5bfc9bebd198e, 0x268c3ba709e04214], - [0x68007bacb2df3316, 0x96e939e46c518d80], - [0xc814e20476a9fb8a, 0x5025c02d59c58239], - [0xde1369676ccc5a71, 0xfa2563959674ee15], - [0x5886ca5d2e2f31d7, 0x7e0af1fa27cf73c3], - [0x749c47ab18501dda, 0xe2757e4f7401905a], - [0xcafaaae3e4d59b34, 0x9adf6acebd10190d], - [0xfe4890d1e6188d0b, 0x046df344706c631e], - ], - ); -} diff --git a/aes/src/x86/vaes256.rs b/aes/src/x86/vaes256.rs deleted file mode 100644 index 5664066fa..000000000 --- a/aes/src/x86/vaes256.rs +++ /dev/null @@ -1 +0,0 @@ -pub(super) mod encdec; diff --git a/aes/src/x86/vaes256/encdec.rs b/aes/src/x86/vaes256/encdec.rs deleted file mode 100644 index 303df83e8..000000000 --- a/aes/src/x86/vaes256/encdec.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::x86::{Block30, Simd128RoundKeys, Simd256RoundKeys, arch::*}; -use cipher::inout::InOut; -use core::mem::MaybeUninit; - -#[target_feature(enable = "avx2")] -#[inline] -pub(crate) unsafe fn broadcast_keys( - keys: &Simd128RoundKeys, -) -> Simd256RoundKeys { - keys.map(|key| _mm256_broadcastsi128_si256(key)) -} - -#[target_feature(enable = "avx2,vaes")] -#[inline] -pub(crate) unsafe fn encrypt30( - keys: &Simd256RoundKeys, - blocks: InOut<'_, '_, Block30>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (iptr, optr) = blocks.into_raw(); - let iptr = iptr.cast::<__m256i>(); - let optr = optr.cast::<__m256i>(); - - let mut data: [MaybeUninit<__m256i>; 15] = unsafe { MaybeUninit::uninit().assume_init() }; - - (0..15).for_each(|i| { - data[i].write(unsafe { iptr.add(i).read_unaligned() }); - }); - let mut data: [__m256i; 15] = unsafe { ::core::mem::transmute(data) }; - - for vec in &mut data { - *vec = _mm256_xor_si256(*vec, keys[0]); - } - for key in &keys[1..KEYS - 1] { - for vec in &mut data { - *vec = _mm256_aesenc_epi128(*vec, *key); - } - } - for vec in &mut data { - *vec = _mm256_aesenclast_epi128(*vec, keys[KEYS - 1]); - } - - (0..15).for_each(|i| { - unsafe { optr.add(i).write_unaligned(data[i]) }; - }); -} - -#[target_feature(enable = "avx2,vaes")] -#[inline] -pub(crate) unsafe fn decrypt30( - keys: &Simd256RoundKeys, - blocks: InOut<'_, '_, Block30>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (iptr, optr) = blocks.into_raw(); - let iptr = iptr.cast::<__m256i>(); - let optr = optr.cast::<__m256i>(); - - let mut data: [MaybeUninit<__m256i>; 15] = unsafe { MaybeUninit::uninit().assume_init() }; - - (0..15).for_each(|i| { - data[i].write(unsafe { iptr.add(i).read_unaligned() }); - }); - let mut data: [__m256i; 15] = unsafe { ::core::mem::transmute(data) }; - - for vec in &mut data { - *vec = _mm256_xor_si256(*vec, keys[0]); - } - for key in &keys[1..KEYS - 1] { - for vec in &mut data { - *vec = _mm256_aesdec_epi128(*vec, *key); - } - } - for vec in &mut data { - *vec = _mm256_aesdeclast_epi128(*vec, keys[KEYS - 1]); - } - - (0..15).for_each(|i| { - unsafe { optr.add(i).write_unaligned(data[i]) }; - }); -} diff --git a/aes/src/x86/vaes512.rs b/aes/src/x86/vaes512.rs deleted file mode 100644 index 5664066fa..000000000 --- a/aes/src/x86/vaes512.rs +++ /dev/null @@ -1 +0,0 @@ -pub(super) mod encdec; diff --git a/aes/src/x86/vaes512/encdec.rs b/aes/src/x86/vaes512/encdec.rs deleted file mode 100644 index f74358a6d..000000000 --- a/aes/src/x86/vaes512/encdec.rs +++ /dev/null @@ -1,85 +0,0 @@ -#![allow(unsafe_op_in_unsafe_fn)] - -use crate::x86::{Block64, Simd128RoundKeys, Simd512RoundKeys, arch::*}; -use cipher::inout::InOut; -use core::mem::MaybeUninit; - -#[target_feature(enable = "avx512f")] -#[inline] -pub(crate) unsafe fn broadcast_keys( - keys: &Simd128RoundKeys, -) -> Simd512RoundKeys { - keys.map(|key| _mm512_broadcast_i32x4(key)) -} - -#[target_feature(enable = "avx512f,vaes")] -#[inline] -pub(crate) unsafe fn encrypt64( - keys: &Simd512RoundKeys, - blocks: InOut<'_, '_, Block64>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (iptr, optr) = blocks.into_raw(); - let iptr = iptr.cast::<__m512i>(); - let optr = optr.cast::<__m512i>(); - - let mut data: [MaybeUninit<__m512i>; 16] = MaybeUninit::uninit().assume_init(); - - (0..16).for_each(|i| { - data[i].write(iptr.add(i).read_unaligned()); - }); - let mut data: [__m512i; 16] = unsafe { ::core::mem::transmute(data) }; - - for vec in &mut data { - *vec = _mm512_xor_si512(*vec, keys[0]); - } - for key in &keys[1..KEYS - 1] { - for vec in &mut data { - *vec = _mm512_aesenc_epi128(*vec, *key); - } - } - for vec in &mut data { - *vec = _mm512_aesenclast_epi128(*vec, keys[KEYS - 1]); - } - - (0..16).for_each(|i| { - optr.add(i).write_unaligned(data[i]); - }); -} - -#[target_feature(enable = "avx512f,vaes")] -#[inline] -pub(crate) unsafe fn decrypt64( - keys: &Simd512RoundKeys, - blocks: InOut<'_, '_, Block64>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - - let (iptr, optr) = blocks.into_raw(); - let iptr = iptr.cast::<__m512i>(); - let optr = optr.cast::<__m512i>(); - - let mut data: [MaybeUninit<__m512i>; 16] = MaybeUninit::uninit().assume_init(); - - (0..16).for_each(|i| { - data[i].write(iptr.add(i).read_unaligned()); - }); - let mut data: [__m512i; 16] = unsafe { ::core::mem::transmute(data) }; - - for vec in &mut data { - *vec = _mm512_xor_si512(*vec, keys[0]); - } - for key in &keys[1..KEYS - 1] { - for vec in &mut data { - *vec = _mm512_aesdec_epi128(*vec, *key); - } - } - for vec in &mut data { - *vec = _mm512_aesdeclast_epi128(*vec, keys[KEYS - 1]); - } - - (0..16).for_each(|i| { - optr.add(i).write_unaligned(data[i]); - }); -} From 7c8cd1219a874bb54deb85f7851e4b9820fb3c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Fri, 24 Jul 2026 18:04:03 +0300 Subject: [PATCH 2/3] Derive `Clone` instead of using manual unsafe impl --- aes/src/lib.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/aes/src/lib.rs b/aes/src/lib.rs index 0f1889456..b3dc34140 100644 --- a/aes/src/lib.rs +++ b/aes/src/lib.rs @@ -390,13 +390,6 @@ macro_rules! impl_from_enc { macro_rules! common_impls { ($name:ident) => { - impl Clone for $name { - #[inline] - fn clone(&self) -> Self { - unsafe { core::ptr::read(self) } - } - } - impl BlockSizeUser for $name { type BlockSize = U16; } @@ -440,6 +433,7 @@ macro_rules! define_aes_impl { mod $module { use crate::backends; + #[derive(Copy, Clone)] pub(super) union Inner { #[cfg(all( any(target_arch = "x86_64", target_arch = "x86"), @@ -451,6 +445,7 @@ macro_rules! define_aes_impl { pub(super) soft: backends::soft::$name, } + #[derive(Copy, Clone)] pub(super) union InnerEnc { #[cfg(all( any(target_arch = "x86_64", target_arch = "x86"), @@ -462,6 +457,7 @@ macro_rules! define_aes_impl { pub(super) soft: backends::soft::$name, } + #[derive(Copy, Clone)] pub(super) union InnerDec { #[cfg(all( any(target_arch = "x86_64", target_arch = "x86"), @@ -476,6 +472,7 @@ macro_rules! define_aes_impl { #[doc=$doc] #[doc = "block cipher"] + #[derive(Clone)] pub struct $name { inner: $module::Inner, #[allow(dead_code, reason = "this field is not used on software-only targets")] @@ -490,6 +487,7 @@ macro_rules! define_aes_impl { #[doc=$doc] #[doc = "block cipher (encrypt-only)"] + #[derive(Clone)] pub struct $name_enc { inner: $module::InnerEnc, #[allow(dead_code, reason = "this field is not used on software-only targets")] @@ -502,6 +500,7 @@ macro_rules! define_aes_impl { #[doc=$doc] #[doc = "block cipher (decrypt-only)"] + #[derive(Clone)] pub struct $name_dec { inner: $module::InnerDec, #[allow(dead_code, reason = "this field is not used on software-only targets")] From 40f81fbf7f73bb869b8a088d5f161a9affbcbb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Fri, 24 Jul 2026 18:11:58 +0300 Subject: [PATCH 3/3] use `pub(super)` in fixslice instead of `pub(crate)` --- aes/src/backends/fixslice/hazmat.rs | 12 +++++------ aes/src/backends/fixslice/mix_columns.rs | 4 ++-- aes/src/backends/fixslice/mod.rs | 14 ++++++------- aes/src/backends/fixslice/sbox.rs | 6 +++--- aes/src/backends/fixslice/utils.rs | 26 ++++++++++++------------ 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/aes/src/backends/fixslice/hazmat.rs b/aes/src/backends/fixslice/hazmat.rs index 5be60a20d..1cddc34a6 100644 --- a/aes/src/backends/fixslice/hazmat.rs +++ b/aes/src/backends/fixslice/hazmat.rs @@ -9,6 +9,12 @@ fn xor_in_place(dst: &mut Block, src: &Block) { } } +#[inline] +fn inv_bitslice_one(block: &mut Block, state: &State) { + let out = W::inv_bitslice(state); + block.copy_from_slice(out[0].as_slice()); +} + fn cipher_round_generic(block: &mut Block, round_key: &Block) { let mut state = State::::default(); W::bitslice(&mut state, &broadcast::(block.as_slice())); @@ -130,9 +136,3 @@ pub(crate) fn mix_columns(block: &mut Block) { pub(crate) fn inv_mix_columns(block: &mut Block) { inv_mix_columns_generic::(block) } - -#[inline] -fn inv_bitslice_one(block: &mut Block, state: &State) { - let out = W::inv_bitslice(state); - block.copy_from_slice(out[0].as_slice()); -} diff --git a/aes/src/backends/fixslice/mix_columns.rs b/aes/src/backends/fixslice/mix_columns.rs index c5a7bc7ba..fdffa0235 100644 --- a/aes/src/backends/fixslice/mix_columns.rs +++ b/aes/src/backends/fixslice/mix_columns.rs @@ -12,7 +12,7 @@ macro_rules! define_mix_columns { $second_rotate:path ) => { #[rustfmt::skip] - pub(crate) fn $name(state: &mut State) { + pub(super) fn $name(state: &mut State) { let (a0, a1, a2, a3, a4, a5, a6, a7) = ( state[0], state[1], state[2], state[3], state[4], state[5], state[6], state[7] ); @@ -47,7 +47,7 @@ macro_rules! define_mix_columns { } #[rustfmt::skip] - pub(crate) fn $name_inv(state: &mut State) { + pub(super) fn $name_inv(state: &mut State) { let (a0, a1, a2, a3, a4, a5, a6, a7) = ( state[0], state[1], state[2], state[3], state[4], state[5], state[6], state[7] ); diff --git a/aes/src/backends/fixslice/mod.rs b/aes/src/backends/fixslice/mod.rs index e4146286d..f5d29d836 100644 --- a/aes/src/backends/fixslice/mod.rs +++ b/aes/src/backends/fixslice/mod.rs @@ -15,9 +15,9 @@ use cipher::array::Array; -pub(crate) mod aes128; -pub(crate) mod aes192; -pub(crate) mod aes256; +pub(super) mod aes128; +pub(super) mod aes192; +pub(super) mod aes256; #[cfg(feature = "hazmat")] pub(crate) mod hazmat; @@ -32,12 +32,12 @@ type State = [W; 8]; cpubits::cpubits! { 16 | 32 => { - pub(crate) type NativeWord = u32; + pub(super) type NativeWord = u32; } 64 => { - pub(crate) type NativeWord = u64; + pub(super) type NativeWord = u64; } } -pub(crate) type NativeBatchSize = ::Blocks; -pub(crate) type BatchBlocks = Array::Blocks>; +pub(super) type NativeBatchSize = ::Blocks; +pub(super) type BatchBlocks = Array::Blocks>; diff --git a/aes/src/backends/fixslice/sbox.rs b/aes/src/backends/fixslice/sbox.rs index b65fcc8c8..2a2e104ac 100644 --- a/aes/src/backends/fixslice/sbox.rs +++ b/aes/src/backends/fixslice/sbox.rs @@ -7,7 +7,7 @@ use super::Word; /// See: /// /// Note that the 4 bitwise NOT are moved to the key schedule. -pub(crate) fn sub_bytes(state: &mut [W]) { +pub(super) fn sub_bytes(state: &mut [W]) { debug_assert_eq!(state.len(), 8); // Scheduled using https://github.com/Ko-/aes-armcortexm/tree/public/scheduler @@ -180,7 +180,7 @@ pub(crate) fn sub_bytes(state: &mut [W]) { /// Note that the 4 bitwise NOT are accounted for here so that it is a true /// inverse of `sub_bytes`. -pub(crate) fn inv_sub_bytes(state: &mut [W]) { +pub(super) fn inv_sub_bytes(state: &mut [W]) { debug_assert_eq!(state.len(), 8); // Scheduled using https://github.com/Ko-/aes-armcortexm/tree/public/scheduler @@ -386,7 +386,7 @@ pub(crate) fn inv_sub_bytes(state: &mut [W]) { /// NOT operations that are omitted in S-box. #[inline] -pub(crate) fn sub_bytes_nots(state: &mut [W]) { +pub(super) fn sub_bytes_nots(state: &mut [W]) { debug_assert_eq!(state.len(), 8); state[0] = !state[0]; state[1] = !state[1]; diff --git a/aes/src/backends/fixslice/utils.rs b/aes/src/backends/fixslice/utils.rs index 637488a35..698225ef7 100644 --- a/aes/src/backends/fixslice/utils.rs +++ b/aes/src/backends/fixslice/utils.rs @@ -4,7 +4,7 @@ use super::{BatchBlocks, State, Word}; /// /// Used by the key schedules, which conceptually call `bitslice(...)` on the /// same input block several times to fill the bitsliced state. -pub(crate) fn broadcast(block: &[u8]) -> BatchBlocks { +pub(super) fn broadcast(block: &[u8]) -> BatchBlocks { debug_assert_eq!(block.len(), 16); let mut out = BatchBlocks::::default(); for slot in out.iter_mut() { @@ -14,13 +14,13 @@ pub(crate) fn broadcast(block: &[u8]) -> BatchBlocks { } #[inline] -fn delta_swap_1(a: &mut W, shift: u32, mask: W) { +pub(super) fn delta_swap_1(a: &mut W, shift: u32, mask: W) { let t = (*a ^ ((*a) >> shift)) & mask; *a ^= t ^ (t << shift); } #[inline] -pub(crate) fn delta_swap_2(a: &mut W, b: &mut W, shift: u32, mask: W) { +pub(super) fn delta_swap_2(a: &mut W, b: &mut W, shift: u32, mask: W) { let t = (*a ^ ((*b) >> shift)) & mask; *a ^= t; *b ^= t << shift; @@ -29,7 +29,7 @@ pub(crate) fn delta_swap_2(a: &mut W, b: &mut W, shift: u32, mask: W) { /// Applies ShiftRows once on an AES state (or key). #[cfg(any(not(aes_backend_soft = "compact"), feature = "hazmat"))] #[inline] -pub(crate) fn shift_rows_1(state: &mut [W]) { +pub(super) fn shift_rows_1(state: &mut [W]) { debug_assert_eq!(state.len(), 8); for x in state.iter_mut() { delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x03, 0x0f, 0x0c)); @@ -39,7 +39,7 @@ pub(crate) fn shift_rows_1(state: &mut [W]) { /// Applies ShiftRows twice on an AES state (or key). #[inline] -pub(crate) fn shift_rows_2(state: &mut [W]) { +pub(super) fn shift_rows_2(state: &mut [W]) { debug_assert_eq!(state.len(), 8); for x in state.iter_mut() { delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x0f, 0x00, 0x0f)); @@ -48,7 +48,7 @@ pub(crate) fn shift_rows_2(state: &mut [W]) { /// Applies ShiftRows three times on an AES state (or key). #[inline] -pub(crate) fn shift_rows_3(state: &mut [W]) { +pub(super) fn shift_rows_3(state: &mut [W]) { debug_assert_eq!(state.len(), 8); for x in state.iter_mut() { delta_swap_1(x, W::HALF_ROW, W::pack_rows(0x00, 0x0c, 0x0f, 0x03)); @@ -57,18 +57,18 @@ pub(crate) fn shift_rows_3(state: &mut [W]) { } #[inline(always)] -pub(crate) fn inv_shift_rows_1(state: &mut [W]) { +pub(super) fn inv_shift_rows_1(state: &mut [W]) { shift_rows_3(state); } #[inline(always)] -pub(crate) fn inv_shift_rows_2(state: &mut [W]) { +pub(super) fn inv_shift_rows_2(state: &mut [W]) { shift_rows_2(state); } #[cfg(not(aes_backend_soft = "compact"))] #[inline(always)] -pub(crate) fn inv_shift_rows_3(state: &mut [W]) { +pub(super) fn inv_shift_rows_3(state: &mut [W]) { shift_rows_1(state); } @@ -80,7 +80,7 @@ pub(crate) fn inv_shift_rows_3(state: &mut [W]) { /// /// The `idx_ror` parameter refers to the rotation value, which varies between the /// different key schedules. -pub(crate) fn xor_columns(rkeys: &mut [W], offset: usize, idx_xor: usize, idx_ror: u32) { +pub(super) fn xor_columns(rkeys: &mut [W], offset: usize, idx_xor: usize, idx_ror: u32) { for i in 0..8 { let off_i = offset + i; let rk = rkeys[off_i - idx_xor] ^ (W::uniform_row(0x03) & rkeys[off_i].ror(idx_ror)); @@ -92,7 +92,7 @@ pub(crate) fn xor_columns(rkeys: &mut [W], offset: usize, idx_xor: usiz } /// Copy 32-bytes within the provided slice to an 8-byte offset. -pub(crate) fn memshift32(buffer: &mut [W], src_offset: usize) { +pub(super) fn memshift32(buffer: &mut [W], src_offset: usize) { debug_assert_eq!(src_offset % 8, 0); let dst_offset = src_offset + 8; @@ -106,7 +106,7 @@ pub(crate) fn memshift32(buffer: &mut [W], src_offset: usize) { /// XOR the round key into the internal state. The round keys are expected /// to be pre-computed and packed in the fixsliced representation. #[inline] -pub(crate) fn add_round_key(state: &mut State, rkey: &[W]) { +pub(super) fn add_round_key(state: &mut State, rkey: &[W]) { debug_assert_eq!(rkey.len(), 8); for (a, b) in state.iter_mut().zip(rkey) { *a ^= *b; @@ -114,6 +114,6 @@ pub(crate) fn add_round_key(state: &mut State, rkey: &[W]) { } #[inline(always)] -pub(crate) fn add_round_constant_bit(state: &mut [W], bit: usize) { +pub(super) fn add_round_constant_bit(state: &mut [W], bit: usize) { state[bit] ^= W::pack_rows(0x00, 0xc0, 0x00, 0x00); }