arch/x86/crypto/chacha_glue.c

   1 /*
   2  * x64 SIMD accelerated ChaCha and XChaCha stream ciphers,
   3  * including ChaCha20 (RFC7539)
   4  *
   5  * Copyright (C) 2015 Martin Willi
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  */
  12
  13 #include <crypto/algapi.h>
  14 #include <crypto/chacha.h>
  15 #include <crypto/internal/skcipher.h>
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <asm/fpu/api.h>
  19 #include <asm/simd.h>
  20
  21 #define CHACHA_STATE_ALIGN 16
  22
  23 asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
  24                                        unsigned int len, int nrounds);
  25 asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
  26                                         unsigned int len, int nrounds);
  27 asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);
  28 #ifdef CONFIG_AS_AVX2
  29 asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
  30                                        unsigned int len, int nrounds);
  31 asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
  32                                        unsigned int len, int nrounds);
  33 asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
  34                                        unsigned int len, int nrounds);
  35 static bool chacha_use_avx2;
  36 #ifdef CONFIG_AS_AVX512
  37 asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
  38                                            unsigned int len, int nrounds);
  39 asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
  40                                            unsigned int len, int nrounds);
  41 asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
  42                                            unsigned int len, int nrounds);
  43 static bool chacha_use_avx512vl;
  44 #endif
  45 #endif
  46
  47 static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks)
  48 {
  49         len = min(len, maxblocks * CHACHA_BLOCK_SIZE);
  50         return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE;
  51 }
  52
  53 static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
  54                           unsigned int bytes, int nrounds)
  55 {
  56 #ifdef CONFIG_AS_AVX2
  57 #ifdef CONFIG_AS_AVX512
  58         if (chacha_use_avx512vl) {
  59                 while (bytes >= CHACHA_BLOCK_SIZE * 8) {
  60                         chacha_8block_xor_avx512vl(state, dst, src, bytes,
  61                                                    nrounds);
  62                         bytes -= CHACHA_BLOCK_SIZE * 8;
  63                         src += CHACHA_BLOCK_SIZE * 8;
  64                         dst += CHACHA_BLOCK_SIZE * 8;
  65                         state[12] += 8;
  66                 }
  67                 if (bytes > CHACHA_BLOCK_SIZE * 4) {
  68                         chacha_8block_xor_avx512vl(state, dst, src, bytes,
  69                                                    nrounds);
  70                         state[12] += chacha_advance(bytes, 8);
  71                         return;
  72                 }
  73                 if (bytes > CHACHA_BLOCK_SIZE * 2) {
  74                         chacha_4block_xor_avx512vl(state, dst, src, bytes,
  75                                                    nrounds);
  76                         state[12] += chacha_advance(bytes, 4);
  77                         return;
  78                 }
  79                 if (bytes) {
  80                         chacha_2block_xor_avx512vl(state, dst, src, bytes,
  81                                                    nrounds);
  82                         state[12] += chacha_advance(bytes, 2);
  83                         return;
  84                 }
  85         }
  86 #endif
  87         if (chacha_use_avx2) {
  88                 while (bytes >= CHACHA_BLOCK_SIZE * 8) {
  89                         chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
  90                         bytes -= CHACHA_BLOCK_SIZE * 8;
  91                         src += CHACHA_BLOCK_SIZE * 8;
  92                         dst += CHACHA_BLOCK_SIZE * 8;
  93                         state[12] += 8;
  94                 }
  95                 if (bytes > CHACHA_BLOCK_SIZE * 4) {
  96                         chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
  97                         state[12] += chacha_advance(bytes, 8);
  98                         return;
  99                 }
 100                 if (bytes > CHACHA_BLOCK_SIZE * 2) {
 101                         chacha_4block_xor_avx2(state, dst, src, bytes, nrounds);
 102                         state[12] += chacha_advance(bytes, 4);
 103                         return;
 104                 }
 105                 if (bytes > CHACHA_BLOCK_SIZE) {
 106                         chacha_2block_xor_avx2(state, dst, src, bytes, nrounds);
 107                         state[12] += chacha_advance(bytes, 2);
 108                         return;
 109                 }
 110         }
 111 #endif
 112         while (bytes >= CHACHA_BLOCK_SIZE * 4) {
 113                 chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
 114                 bytes -= CHACHA_BLOCK_SIZE * 4;
 115                 src += CHACHA_BLOCK_SIZE * 4;
 116                 dst += CHACHA_BLOCK_SIZE * 4;
 117                 state[12] += 4;
 118         }
 119         if (bytes > CHACHA_BLOCK_SIZE) {
 120                 chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
 121                 state[12] += chacha_advance(bytes, 4);
 122                 return;
 123         }
 124         if (bytes) {
 125                 chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);
 126                 state[12]++;
 127         }
 128 }
 129
 130 static int chacha_simd_stream_xor(struct skcipher_request *req,
 131                                   struct chacha_ctx *ctx, u8 *iv)
 132 {
 133         u32 *state, state_buf[16 + 2] __aligned(8);
 134         struct skcipher_walk walk;
 135         int err;
 136
 137         BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
 138         state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);
 139
 140         err = skcipher_walk_virt(&walk, req, true);
 141
 142         crypto_chacha_init(state, ctx, iv);
 143
 144         while (walk.nbytes > 0) {
 145                 unsigned int nbytes = walk.nbytes;
 146
 147                 if (nbytes < walk.total)
 148                         nbytes = round_down(nbytes, walk.stride);
 149
 150                 chacha_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
 151                               nbytes, ctx->nrounds);
 152
 153                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 154         }
 155
 156         return err;
 157 }
 158
 159 static int chacha_simd(struct skcipher_request *req)
 160 {
 161         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 162         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 163         int err;
 164
 165         if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable())
 166                 return crypto_chacha_crypt(req);
 167
 168         kernel_fpu_begin();
 169         err = chacha_simd_stream_xor(req, ctx, req->iv);
 170         kernel_fpu_end();
 171         return err;
 172 }
 173
 174 static int xchacha_simd(struct skcipher_request *req)
 175 {
 176         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 177         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 178         struct chacha_ctx subctx;
 179         u32 *state, state_buf[16 + 2] __aligned(8);
 180         u8 real_iv[16];
 181         int err;
 182
 183         if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable())
 184                 return crypto_xchacha_crypt(req);
 185
 186         BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
 187         state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);
 188         crypto_chacha_init(state, ctx, req->iv);
 189
 190         kernel_fpu_begin();
 191
 192         hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
 193         subctx.nrounds = ctx->nrounds;
 194
 195         memcpy(&real_iv[0], req->iv + 24, 8);
 196         memcpy(&real_iv[8], req->iv + 16, 8);
 197         err = chacha_simd_stream_xor(req, &subctx, real_iv);
 198
 199         kernel_fpu_end();
 200
 201         return err;
 202 }
 203
 204 static struct skcipher_alg algs[] = {
 205         {
 206                 .base.cra_name          = "chacha20",
 207                 .base.cra_driver_name   = "chacha20-simd",
 208                 .base.cra_priority      = 300,
 209                 .base.cra_blocksize     = 1,
 210                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 211                 .base.cra_module        = THIS_MODULE,
 212
 213                 .min_keysize            = CHACHA_KEY_SIZE,
 214                 .max_keysize            = CHACHA_KEY_SIZE,
 215                 .ivsize                 = CHACHA_IV_SIZE,
 216                 .chunksize              = CHACHA_BLOCK_SIZE,
 217                 .setkey                 = crypto_chacha20_setkey,
 218                 .encrypt                = chacha_simd,
 219                 .decrypt                = chacha_simd,
 220         }, {
 221                 .base.cra_name          = "xchacha20",
 222                 .base.cra_driver_name   = "xchacha20-simd",
 223                 .base.cra_priority      = 300,
 224                 .base.cra_blocksize     = 1,
 225                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 226                 .base.cra_module        = THIS_MODULE,
 227
 228                 .min_keysize            = CHACHA_KEY_SIZE,
 229                 .max_keysize            = CHACHA_KEY_SIZE,
 230                 .ivsize                 = XCHACHA_IV_SIZE,
 231                 .chunksize              = CHACHA_BLOCK_SIZE,
 232                 .setkey                 = crypto_chacha20_setkey,
 233                 .encrypt                = xchacha_simd,
 234                 .decrypt                = xchacha_simd,
 235         },
 236 };
 237
 238 static int __init chacha_simd_mod_init(void)
 239 {
 240         if (!boot_cpu_has(X86_FEATURE_SSSE3))
 241                 return -ENODEV;
 242
 243 #ifdef CONFIG_AS_AVX2
 244         chacha_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
 245                           boot_cpu_has(X86_FEATURE_AVX2) &&
 246                           cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
 247 #ifdef CONFIG_AS_AVX512
 248         chacha_use_avx512vl = chacha_use_avx2 &&
 249                               boot_cpu_has(X86_FEATURE_AVX512VL) &&
 250                               boot_cpu_has(X86_FEATURE_AVX512BW); /* kmovq */
 251 #endif
 252 #endif
 253         return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
 254 }
 255
 256 static void __exit chacha_simd_mod_fini(void)
 257 {
 258         crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 259 }
 260
 261 module_init(chacha_simd_mod_init);
 262 module_exit(chacha_simd_mod_fini);
 263
 264 MODULE_LICENSE("GPL");
 265 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
 266 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (x64 SIMD accelerated)");
 267 MODULE_ALIAS_CRYPTO("chacha20");
 268 MODULE_ALIAS_CRYPTO("chacha20-simd");
 269 MODULE_ALIAS_CRYPTO("xchacha20");
 270 MODULE_ALIAS_CRYPTO("xchacha20-simd");