crypto: arm64/aes - implement accelerated ESSIV/CBC mode

Add an accelerated version of the 'essiv(cbc(aes),sha256)' skcipher,
which is used by fscrypt or dm-crypt on systems where CBC mode is
signficantly more performant than XTS mode (e.g., when using a h/w
accelerator which supports the former but not the latter) This avoids
a separate call into the AES cipher for every invocation.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 23abf33..ca0c84d 100644 (file)
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -10,6 +10,7 @@
 #include <asm/simd.h>
 #include <crypto/aes.h>
 #include <crypto/ctr.h>
+#include <crypto/sha.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
@@ -30,6 +31,8 @@
 #define aes_cbc_decrypt                ce_aes_cbc_decrypt
 #define aes_cbc_cts_encrypt    ce_aes_cbc_cts_encrypt
 #define aes_cbc_cts_decrypt    ce_aes_cbc_cts_decrypt
+#define aes_essiv_cbc_encrypt  ce_aes_essiv_cbc_encrypt
+#define aes_essiv_cbc_decrypt  ce_aes_essiv_cbc_decrypt
 #define aes_ctr_encrypt                ce_aes_ctr_encrypt
 #define aes_xts_encrypt                ce_aes_xts_encrypt
 #define aes_xts_decrypt                ce_aes_xts_decrypt
@@ -44,6 +47,8 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
 #define aes_cbc_decrypt                neon_aes_cbc_decrypt
 #define aes_cbc_cts_encrypt    neon_aes_cbc_cts_encrypt
 #define aes_cbc_cts_decrypt    neon_aes_cbc_cts_decrypt
+#define aes_essiv_cbc_encrypt  neon_aes_essiv_cbc_encrypt
+#define aes_essiv_cbc_decrypt  neon_aes_essiv_cbc_decrypt
 #define aes_ctr_encrypt                neon_aes_ctr_encrypt
 #define aes_xts_encrypt                neon_aes_xts_encrypt
 #define aes_xts_decrypt                neon_aes_xts_decrypt
@@ -51,6 +56,7 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
 MODULE_ALIAS_CRYPTO("ecb(aes)");
 MODULE_ALIAS_CRYPTO("cbc(aes)");
+MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
 MODULE_ALIAS_CRYPTO("ctr(aes)");
 MODULE_ALIAS_CRYPTO("xts(aes)");
 MODULE_ALIAS_CRYPTO("cmac(aes)");
@@ -87,6 +93,13 @@ asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
                                int rounds, int blocks, u32 const rk2[], u8 iv[],
                                int first);
 
+asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+                                     int rounds, int blocks, u8 iv[],
+                                     u32 const rk2[]);
+asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+                                     int rounds, int blocks, u8 iv[],
+                                     u32 const rk2[]);
+
 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
                               int blocks, u8 dg[], int enc_before,
                               int enc_after);
@@ -102,6 +115,12 @@ struct crypto_aes_xts_ctx {
        struct crypto_aes_ctx __aligned(8) key2;
 };
 
+struct crypto_aes_essiv_cbc_ctx {
+       struct crypto_aes_ctx key1;
+       struct crypto_aes_ctx __aligned(8) key2;
+       struct crypto_shash *hash;
+};
+
 struct mac_tfm_ctx {
        struct crypto_aes_ctx key;
        u8 __aligned(8) consts[];
@@ -146,6 +165,31 @@ static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
        return -EINVAL;
 }
 
+static int essiv_cbc_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
+                            unsigned int key_len)
+{
+       struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+       SHASH_DESC_ON_STACK(desc, ctx->hash);
+       u8 digest[SHA256_DIGEST_SIZE];
+       int ret;
+
+       ret = aes_expandkey(&ctx->key1, in_key, key_len);
+       if (ret)
+               goto out;
+
+       desc->tfm = ctx->hash;
+       crypto_shash_digest(desc, in_key, key_len, digest);
+
+       ret = aes_expandkey(&ctx->key2, digest, sizeof(digest));
+       if (ret)
+               goto out;
+
+       return 0;
+out:
+       crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+       return -EINVAL;
+}
+
 static int ecb_encrypt(struct skcipher_request *req)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -360,6 +404,68 @@ static int cts_cbc_decrypt(struct skcipher_request *req)
        return skcipher_walk_done(&walk, 0);
 }
 
+static int essiv_cbc_init_tfm(struct crypto_skcipher *tfm)
+{
+       struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+       ctx->hash = crypto_alloc_shash("sha256", 0, 0);
+       if (IS_ERR(ctx->hash))
+               return PTR_ERR(ctx->hash);
+
+       return 0;
+}
+
+static void essiv_cbc_exit_tfm(struct crypto_skcipher *tfm)
+{
+       struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+       crypto_free_shash(ctx->hash);
+}
+
+static int essiv_cbc_encrypt(struct skcipher_request *req)
+{
+       struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+       struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+       int err, rounds = 6 + ctx->key1.key_length / 4;
+       struct skcipher_walk walk;
+       unsigned int blocks;
+
+       err = skcipher_walk_virt(&walk, req, false);
+
+       blocks = walk.nbytes / AES_BLOCK_SIZE;
+       if (blocks) {
+               kernel_neon_begin();
+               aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                                     ctx->key1.key_enc, rounds, blocks,
+                                     req->iv, ctx->key2.key_enc);
+               kernel_neon_end();
+               err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
+       }
+       return err ?: cbc_encrypt_walk(req, &walk);
+}
+
+static int essiv_cbc_decrypt(struct skcipher_request *req)
+{
+       struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+       struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+       int err, rounds = 6 + ctx->key1.key_length / 4;
+       struct skcipher_walk walk;
+       unsigned int blocks;
+
+       err = skcipher_walk_virt(&walk, req, false);
+
+       blocks = walk.nbytes / AES_BLOCK_SIZE;
+       if (blocks) {
+               kernel_neon_begin();
+               aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                                     ctx->key1.key_dec, rounds, blocks,
+                                     req->iv, ctx->key2.key_enc);
+               kernel_neon_end();
+               err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
+       }
+       return err ?: cbc_decrypt_walk(req, &walk);
+}
+
 static int ctr_encrypt(struct skcipher_request *req)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -517,6 +623,24 @@ static struct skcipher_alg aes_algs[] = { {
        .init           = cts_cbc_init_tfm,
 }, {
        .base = {
+               .cra_name               = "__essiv(cbc(aes),sha256)",
+               .cra_driver_name        = "__essiv-cbc-aes-sha256-" MODE,
+               .cra_priority           = PRIO + 1,
+               .cra_flags              = CRYPTO_ALG_INTERNAL,
+               .cra_blocksize          = AES_BLOCK_SIZE,
+               .cra_ctxsize            = sizeof(struct crypto_aes_essiv_cbc_ctx),
+               .cra_module             = THIS_MODULE,
+       },
+       .min_keysize    = AES_MIN_KEY_SIZE,
+       .max_keysize    = AES_MAX_KEY_SIZE,
+       .ivsize         = AES_BLOCK_SIZE,
+       .setkey         = essiv_cbc_set_key,
+       .encrypt        = essiv_cbc_encrypt,
+       .decrypt        = essiv_cbc_decrypt,
+       .init           = essiv_cbc_init_tfm,
+       .exit           = essiv_cbc_exit_tfm,
+}, {
+       .base = {
                .cra_name               = "__ctr(aes)",
                .cra_driver_name        = "__ctr-aes-" MODE,
                .cra_priority           = PRIO,

diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index 324039b..2879f03 100644 (file)
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -118,8 +118,23 @@ AES_ENDPROC(aes_ecb_decrypt)
         *                 int blocks, u8 iv[])
         * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
         *                 int blocks, u8 iv[])
+        * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+        *                       int rounds, int blocks, u8 iv[],
+        *                       u32 const rk2[]);
+        * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+        *                       int rounds, int blocks, u8 iv[],
+        *                       u32 const rk2[]);
         */
 
+AES_ENTRY(aes_essiv_cbc_encrypt)
+       ld1             {v4.16b}, [x5]                  /* get iv */
+
+       mov             w8, #14                         /* AES-256: 14 rounds */
+       enc_prepare     w8, x6, x7
+       encrypt_block   v4, w8, x6, x7, w9
+       enc_switch_key  w3, x2, x6
+       b               .Lcbcencloop4x
+
 AES_ENTRY(aes_cbc_encrypt)
        ld1             {v4.16b}, [x5]                  /* get iv */
        enc_prepare     w3, x2, x6
@@ -153,13 +168,25 @@ AES_ENTRY(aes_cbc_encrypt)
        st1             {v4.16b}, [x5]                  /* return iv */
        ret
 AES_ENDPROC(aes_cbc_encrypt)
+AES_ENDPROC(aes_essiv_cbc_encrypt)
+
+AES_ENTRY(aes_essiv_cbc_decrypt)
+       stp             x29, x30, [sp, #-16]!
+       mov             x29, sp
+
+       ld1             {cbciv.16b}, [x5]               /* get iv */
 
+       mov             w8, #14                         /* AES-256: 14 rounds */
+       enc_prepare     w8, x6, x7
+       encrypt_block   cbciv, w8, x6, x7, w9
+       b               .Lessivcbcdecstart
 
 AES_ENTRY(aes_cbc_decrypt)
        stp             x29, x30, [sp, #-16]!
        mov             x29, sp
 
        ld1             {cbciv.16b}, [x5]               /* get iv */
+.Lessivcbcdecstart:
        dec_prepare     w3, x2, x6
 
 .LcbcdecloopNx:
@@ -212,6 +239,7 @@ ST5(        st1             {v4.16b}, [x0], #16             )
        ldp             x29, x30, [sp], #16
        ret
 AES_ENDPROC(aes_cbc_decrypt)
+AES_ENDPROC(aes_essiv_cbc_decrypt)
 
 
        /*