lib/crypto/chacha20poly1305.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  *
   5  * This is an implementation of the ChaCha20Poly1305 AEAD construction.
   6  *
   7  * Information: https://tools.ietf.org/html/rfc8439
   8  */
   9
  10 #include <crypto/algapi.h>
  11 #include <crypto/chacha20poly1305.h>
  12 #include <crypto/chacha.h>
  13 #include <crypto/poly1305.h>
  14 #include <crypto/scatterwalk.h>
  15
  16 #include <asm/unaligned.h>
  17 #include <linux/kernel.h>
  18 #include <linux/init.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21
  22 #define CHACHA_KEY_WORDS        (CHACHA_KEY_SIZE / sizeof(u32))
  23
  24 bool __init chacha20poly1305_selftest(void);
  25
  26 static void chacha_load_key(u32 *k, const u8 *in)
  27 {
  28         k[0] = get_unaligned_le32(in);
  29         k[1] = get_unaligned_le32(in + 4);
  30         k[2] = get_unaligned_le32(in + 8);
  31         k[3] = get_unaligned_le32(in + 12);
  32         k[4] = get_unaligned_le32(in + 16);
  33         k[5] = get_unaligned_le32(in + 20);
  34         k[6] = get_unaligned_le32(in + 24);
  35         k[7] = get_unaligned_le32(in + 28);
  36 }
  37
  38 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
  39 {
  40         u32 k[CHACHA_KEY_WORDS];
  41         u8 iv[CHACHA_IV_SIZE];
  42
  43         memset(iv, 0, 8);
  44         memcpy(iv + 8, nonce + 16, 8);
  45
  46         chacha_load_key(k, key);
  47
  48         /* Compute the subkey given the original key and first 128 nonce bits */
  49         chacha_init(chacha_state, k, nonce);
  50         hchacha_block(chacha_state, k, 20);
  51
  52         chacha_init(chacha_state, k, iv);
  53
  54         memzero_explicit(k, sizeof(k));
  55         memzero_explicit(iv, sizeof(iv));
  56 }
  57
  58 static void
  59 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  60                            const u8 *ad, const size_t ad_len, u32 *chacha_state)
  61 {
  62         const u8 *pad0 = page_address(ZERO_PAGE(0));
  63         struct poly1305_desc_ctx poly1305_state;
  64         union {
  65                 u8 block0[POLY1305_KEY_SIZE];
  66                 __le64 lens[2];
  67         } b;
  68
  69         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
  70         poly1305_init(&poly1305_state, b.block0);
  71
  72         poly1305_update(&poly1305_state, ad, ad_len);
  73         if (ad_len & 0xf)
  74                 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
  75
  76         chacha20_crypt(chacha_state, dst, src, src_len);
  77
  78         poly1305_update(&poly1305_state, dst, src_len);
  79         if (src_len & 0xf)
  80                 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
  81
  82         b.lens[0] = cpu_to_le64(ad_len);
  83         b.lens[1] = cpu_to_le64(src_len);
  84         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
  85
  86         poly1305_final(&poly1305_state, dst + src_len);
  87
  88         memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
  89         memzero_explicit(&b, sizeof(b));
  90 }
  91
  92 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  93                               const u8 *ad, const size_t ad_len,
  94                               const u64 nonce,
  95                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
  96 {
  97         u32 chacha_state[CHACHA_STATE_WORDS];
  98         u32 k[CHACHA_KEY_WORDS];
  99         __le64 iv[2];
 100
 101         chacha_load_key(k, key);
 102
 103         iv[0] = 0;
 104         iv[1] = cpu_to_le64(nonce);
 105
 106         chacha_init(chacha_state, k, (u8 *)iv);
 107         __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
 108
 109         memzero_explicit(iv, sizeof(iv));
 110         memzero_explicit(k, sizeof(k));
 111 }
 112 EXPORT_SYMBOL(chacha20poly1305_encrypt);
 113
 114 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 115                                const u8 *ad, const size_t ad_len,
 116                                const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
 117                                const u8 key[CHACHA20POLY1305_KEY_SIZE])
 118 {
 119         u32 chacha_state[CHACHA_STATE_WORDS];
 120
 121         xchacha_init(chacha_state, key, nonce);
 122         __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
 123 }
 124 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
 125
 126 static bool
 127 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 128                            const u8 *ad, const size_t ad_len, u32 *chacha_state)
 129 {
 130         const u8 *pad0 = page_address(ZERO_PAGE(0));
 131         struct poly1305_desc_ctx poly1305_state;
 132         size_t dst_len;
 133         int ret;
 134         union {
 135                 u8 block0[POLY1305_KEY_SIZE];
 136                 u8 mac[POLY1305_DIGEST_SIZE];
 137                 __le64 lens[2];
 138         } b;
 139
 140         if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 141                 return false;
 142
 143         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 144         poly1305_init(&poly1305_state, b.block0);
 145
 146         poly1305_update(&poly1305_state, ad, ad_len);
 147         if (ad_len & 0xf)
 148                 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 149
 150         dst_len = src_len - POLY1305_DIGEST_SIZE;
 151         poly1305_update(&poly1305_state, src, dst_len);
 152         if (dst_len & 0xf)
 153                 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
 154
 155         b.lens[0] = cpu_to_le64(ad_len);
 156         b.lens[1] = cpu_to_le64(dst_len);
 157         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 158
 159         poly1305_final(&poly1305_state, b.mac);
 160
 161         ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
 162         if (likely(!ret))
 163                 chacha20_crypt(chacha_state, dst, src, dst_len);
 164
 165         memzero_explicit(&b, sizeof(b));
 166
 167         return !ret;
 168 }
 169
 170 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 171                               const u8 *ad, const size_t ad_len,
 172                               const u64 nonce,
 173                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
 174 {
 175         u32 chacha_state[CHACHA_STATE_WORDS];
 176         u32 k[CHACHA_KEY_WORDS];
 177         __le64 iv[2];
 178         bool ret;
 179
 180         chacha_load_key(k, key);
 181
 182         iv[0] = 0;
 183         iv[1] = cpu_to_le64(nonce);
 184
 185         chacha_init(chacha_state, k, (u8 *)iv);
 186         ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
 187                                          chacha_state);
 188
 189         memzero_explicit(chacha_state, sizeof(chacha_state));
 190         memzero_explicit(iv, sizeof(iv));
 191         memzero_explicit(k, sizeof(k));
 192         return ret;
 193 }
 194 EXPORT_SYMBOL(chacha20poly1305_decrypt);
 195
 196 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 197                                const u8 *ad, const size_t ad_len,
 198                                const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
 199                                const u8 key[CHACHA20POLY1305_KEY_SIZE])
 200 {
 201         u32 chacha_state[CHACHA_STATE_WORDS];
 202
 203         xchacha_init(chacha_state, key, nonce);
 204         return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
 205                                           chacha_state);
 206 }
 207 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
 208
 209 static
 210 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
 211                                        const size_t src_len,
 212                                        const u8 *ad, const size_t ad_len,
 213                                        const u64 nonce,
 214                                        const u8 key[CHACHA20POLY1305_KEY_SIZE],
 215                                        int encrypt)
 216 {
 217         const u8 *pad0 = page_address(ZERO_PAGE(0));
 218         struct poly1305_desc_ctx poly1305_state;
 219         u32 chacha_state[CHACHA_STATE_WORDS];
 220         struct sg_mapping_iter miter;
 221         size_t partial = 0;
 222         unsigned int flags;
 223         bool ret = true;
 224         int sl;
 225         union {
 226                 struct {
 227                         u32 k[CHACHA_KEY_WORDS];
 228                         __le64 iv[2];
 229                 };
 230                 u8 block0[POLY1305_KEY_SIZE];
 231                 u8 chacha_stream[CHACHA_BLOCK_SIZE];
 232                 struct {
 233                         u8 mac[2][POLY1305_DIGEST_SIZE];
 234                 };
 235                 __le64 lens[2];
 236         } b __aligned(16);
 237
 238         chacha_load_key(b.k, key);
 239
 240         b.iv[0] = 0;
 241         b.iv[1] = cpu_to_le64(nonce);
 242
 243         chacha_init(chacha_state, b.k, (u8 *)b.iv);
 244         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 245         poly1305_init(&poly1305_state, b.block0);
 246
 247         if (unlikely(ad_len)) {
 248                 poly1305_update(&poly1305_state, ad, ad_len);
 249                 if (ad_len & 0xf)
 250                         poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 251         }
 252
 253         flags = SG_MITER_TO_SG;
 254         if (!preemptible())
 255                 flags |= SG_MITER_ATOMIC;
 256
 257         sg_miter_start(&miter, src, sg_nents(src), flags);
 258
 259         for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
 260                 u8 *addr = miter.addr;
 261                 size_t length = min_t(size_t, sl, miter.length);
 262
 263                 if (!encrypt)
 264                         poly1305_update(&poly1305_state, addr, length);
 265
 266                 if (unlikely(partial)) {
 267                         size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
 268
 269                         crypto_xor(addr, b.chacha_stream + partial, l);
 270                         partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
 271
 272                         addr += l;
 273                         length -= l;
 274                 }
 275
 276                 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
 277                         size_t l = length;
 278
 279                         if (unlikely(length < sl))
 280                                 l &= ~(CHACHA_BLOCK_SIZE - 1);
 281                         chacha20_crypt(chacha_state, addr, addr, l);
 282                         addr += l;
 283                         length -= l;
 284                 }
 285
 286                 if (unlikely(length > 0)) {
 287                         chacha20_crypt(chacha_state, b.chacha_stream, pad0,
 288                                        CHACHA_BLOCK_SIZE);
 289                         crypto_xor(addr, b.chacha_stream, length);
 290                         partial = length;
 291                 }
 292
 293                 if (encrypt)
 294                         poly1305_update(&poly1305_state, miter.addr,
 295                                         min_t(size_t, sl, miter.length));
 296         }
 297
 298         if (src_len & 0xf)
 299                 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
 300
 301         b.lens[0] = cpu_to_le64(ad_len);
 302         b.lens[1] = cpu_to_le64(src_len);
 303         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 304
 305         if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
 306                 if (encrypt) {
 307                         poly1305_final(&poly1305_state,
 308                                        miter.addr + miter.length + sl);
 309                         ret = true;
 310                 } else {
 311                         poly1305_final(&poly1305_state, b.mac[0]);
 312                         ret = !crypto_memneq(b.mac[0],
 313                                              miter.addr + miter.length + sl,
 314                                              POLY1305_DIGEST_SIZE);
 315                 }
 316         }
 317
 318         sg_miter_stop(&miter);
 319
 320         if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
 321                 poly1305_final(&poly1305_state, b.mac[1]);
 322                 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
 323                                          sizeof(b.mac[1]), encrypt);
 324                 ret = encrypt ||
 325                       !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
 326         }
 327
 328         memzero_explicit(chacha_state, sizeof(chacha_state));
 329         memzero_explicit(&b, sizeof(b));
 330
 331         return ret;
 332 }
 333
 334 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 335                                          const u8 *ad, const size_t ad_len,
 336                                          const u64 nonce,
 337                                          const u8 key[CHACHA20POLY1305_KEY_SIZE])
 338 {
 339         return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
 340                                                  nonce, key, 1);
 341 }
 342 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
 343
 344 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 345                                          const u8 *ad, const size_t ad_len,
 346                                          const u64 nonce,
 347                                          const u8 key[CHACHA20POLY1305_KEY_SIZE])
 348 {
 349         if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 350                 return false;
 351
 352         return chacha20poly1305_crypt_sg_inplace(src,
 353                                                  src_len - POLY1305_DIGEST_SIZE,
 354                                                  ad, ad_len, nonce, key, 0);
 355 }
 356 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
 357
 358 static int __init mod_init(void)
 359 {
 360         if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
 361             WARN_ON(!chacha20poly1305_selftest()))
 362                 return -ENODEV;
 363         return 0;
 364 }
 365
 366 module_init(mod_init);
 367 MODULE_LICENSE("GPL v2");
 368 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
 369 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");