crypto: aegis128 - add support for SIMD acceleration

Add some plumbing to allow the AEGIS128 code to be built with SIMD
routines for acceleration.

Reviewed-by: Ondrej Mosnacek <omosnace@redhat.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/crypto/Makefile b/crypto/Makefile
index 93375e1..362a36f 100644 (file)
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -90,6 +90,7 @@ obj-$(CONFIG_CRYPTO_GCM) += gcm.o
 obj-$(CONFIG_CRYPTO_CCM) += ccm.o
 obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o
 obj-$(CONFIG_CRYPTO_AEGIS128) += aegis128.o
+aegis128-y := aegis128-core.o
 obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
 obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
 obj-$(CONFIG_CRYPTO_DES) += des_generic.o

diff --git a/crypto/aegis.h b/crypto/aegis.h
index 3308066..6cb65a4 100644 (file)
--- a/crypto/aegis.h
+++ b/crypto/aegis.h
@@ -35,23 +35,23 @@ static const union aegis_block crypto_aegis_const[2] = {
        } },
 };
 
-static void crypto_aegis_block_xor(union aegis_block *dst,
-                                  const union aegis_block *src)
+static inline void crypto_aegis_block_xor(union aegis_block *dst,
+                                         const union aegis_block *src)
 {
        dst->words64[0] ^= src->words64[0];
        dst->words64[1] ^= src->words64[1];
 }
 
-static void crypto_aegis_block_and(union aegis_block *dst,
-                                  const union aegis_block *src)
+static inline void crypto_aegis_block_and(union aegis_block *dst,
+                                         const union aegis_block *src)
 {
        dst->words64[0] &= src->words64[0];
        dst->words64[1] &= src->words64[1];
 }
 
-static void crypto_aegis_aesenc(union aegis_block *dst,
-                               const union aegis_block *src,
-                               const union aegis_block *key)
+static inline void crypto_aegis_aesenc(union aegis_block *dst,
+                                      const union aegis_block *src,
+                                      const union aegis_block *key)
 {
        const u8  *s  = src->bytes;
        const u32 *t = crypto_ft_tab[0];

diff --git a/crypto/aegis128.c b/crypto/aegis128-core.c
similarity index 89%
rename from crypto/aegis128.c
rename to crypto/aegis128-core.c
index 32840d5..f815b46 100644 (file)
--- a/crypto/aegis128.c
+++ b/crypto/aegis128-core.c
@@ -8,6 +8,7 @@
 
 #include <crypto/algapi.h>
 #include <crypto/internal/aead.h>
+#include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
@@ -15,6 +16,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
+#include <asm/simd.h>
 
 #include "aegis.h"
 
@@ -40,6 +42,15 @@ struct aegis128_ops {
                            const u8 *src, unsigned int size);
 };
 
+static bool have_simd;
+
+bool crypto_aegis128_have_simd(void);
+void crypto_aegis128_update_simd(struct aegis_state *state, const void *msg);
+void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst,
+                                       const u8 *src, unsigned int size);
+void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst,
+                                       const u8 *src, unsigned int size);
+
 static void crypto_aegis128_update(struct aegis_state *state)
 {
        union aegis_block tmp;
@@ -55,12 +66,22 @@ static void crypto_aegis128_update(struct aegis_state *state)
 static void crypto_aegis128_update_a(struct aegis_state *state,
                                     const union aegis_block *msg)
 {
+       if (have_simd && crypto_simd_usable()) {
+               crypto_aegis128_update_simd(state, msg);
+               return;
+       }
+
        crypto_aegis128_update(state);
        crypto_aegis_block_xor(&state->blocks[0], msg);
 }
 
 static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg)
 {
+       if (have_simd && crypto_simd_usable()) {
+               crypto_aegis128_update_simd(state, msg);
+               return;
+       }
+
        crypto_aegis128_update(state);
        crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
 }
@@ -365,7 +386,7 @@ static void crypto_aegis128_crypt(struct aead_request *req,
 
 static int crypto_aegis128_encrypt(struct aead_request *req)
 {
-       static const struct aegis128_ops ops = {
+       const struct aegis128_ops *ops = &(struct aegis128_ops){
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_chunk = crypto_aegis128_encrypt_chunk,
        };
@@ -375,7 +396,12 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;
 
-       crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
+       if (have_simd && crypto_simd_usable())
+               ops = &(struct aegis128_ops){
+                       .skcipher_walk_init = skcipher_walk_aead_encrypt,
+                       .crypt_chunk = crypto_aegis128_encrypt_chunk_simd };
+
+       crypto_aegis128_crypt(req, &tag, cryptlen, ops);
 
        scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
                                 authsize, 1);
@@ -384,7 +410,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
 
 static int crypto_aegis128_decrypt(struct aead_request *req)
 {
-       static const struct aegis128_ops ops = {
+       const struct aegis128_ops *ops = &(struct aegis128_ops){
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_chunk = crypto_aegis128_decrypt_chunk,
        };
@@ -398,7 +424,12 @@ static int crypto_aegis128_decrypt(struct aead_request *req)
        scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
                                 authsize, 0);
 
-       crypto_aegis128_crypt(req, &tag, cryptlen, &ops);
+       if (have_simd && crypto_simd_usable())
+               ops = &(struct aegis128_ops){
+                       .skcipher_walk_init = skcipher_walk_aead_decrypt,
+                       .crypt_chunk = crypto_aegis128_decrypt_chunk_simd };
+
+       crypto_aegis128_crypt(req, &tag, cryptlen, ops);
 
        return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
 }
@@ -429,6 +460,9 @@ static struct aead_alg crypto_aegis128_alg = {
 
 static int __init crypto_aegis128_module_init(void)
 {
+       if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD))
+               have_simd = crypto_aegis128_have_simd();
+
        return crypto_register_aead(&crypto_aegis128_alg);
 }