drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_address.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/io.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/rtnetlink.h>
  42 #include <linux/slab.h>
  43
  44 #include <crypto/algapi.h>
  45 #include <crypto/aes.h>
  46 #include <crypto/des.h>
  47 #include <crypto/sha.h>
  48 #include <crypto/md5.h>
  49 #include <crypto/internal/aead.h>
  50 #include <crypto/authenc.h>
  51 #include <crypto/skcipher.h>
  52 #include <crypto/hash.h>
  53 #include <crypto/internal/hash.h>
  54 #include <crypto/scatterwalk.h>
  55
  56 #include "talitos.h"
  57
  58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
  59                            bool is_sec1)
  60 {
  61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
  62         if (!is_sec1)
  63                 ptr->eptr = upper_32_bits(dma_addr);
  64 }
  65
  66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
  67                              struct talitos_ptr *src_ptr, bool is_sec1)
  68 {
  69         dst_ptr->ptr = src_ptr->ptr;
  70         if (!is_sec1)
  71                 dst_ptr->eptr = src_ptr->eptr;
  72 }
  73
  74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
  75                                bool is_sec1)
  76 {
  77         if (is_sec1) {
  78                 ptr->res = 0;
  79                 ptr->len1 = cpu_to_be16(len);
  80         } else {
  81                 ptr->len = cpu_to_be16(len);
  82         }
  83 }
  84
  85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
  86                                            bool is_sec1)
  87 {
  88         if (is_sec1)
  89                 return be16_to_cpu(ptr->len1);
  90         else
  91                 return be16_to_cpu(ptr->len);
  92 }
  93
  94 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
  95 {
  96         if (!is_sec1)
  97                 ptr->j_extent = 0;
  98 }
  99
 100 /*
 101  * map virtual single (contiguous) pointer to h/w descriptor pointer
 102  */
 103 static void map_single_talitos_ptr(struct device *dev,
 104                                    struct talitos_ptr *ptr,
 105                                    unsigned int len, void *data,
 106                                    enum dma_data_direction dir)
 107 {
 108         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
 109         struct talitos_private *priv = dev_get_drvdata(dev);
 110         bool is_sec1 = has_ftr_sec1(priv);
 111
 112         to_talitos_ptr_len(ptr, len, is_sec1);
 113         to_talitos_ptr(ptr, dma_addr, is_sec1);
 114         to_talitos_ptr_extent_clear(ptr, is_sec1);
 115 }
 116
 117 /*
 118  * unmap bus single (contiguous) h/w descriptor pointer
 119  */
 120 static void unmap_single_talitos_ptr(struct device *dev,
 121                                      struct talitos_ptr *ptr,
 122                                      enum dma_data_direction dir)
 123 {
 124         struct talitos_private *priv = dev_get_drvdata(dev);
 125         bool is_sec1 = has_ftr_sec1(priv);
 126
 127         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
 128                          from_talitos_ptr_len(ptr, is_sec1), dir);
 129 }
 130
 131 static int reset_channel(struct device *dev, int ch)
 132 {
 133         struct talitos_private *priv = dev_get_drvdata(dev);
 134         unsigned int timeout = TALITOS_TIMEOUT;
 135         bool is_sec1 = has_ftr_sec1(priv);
 136
 137         if (is_sec1) {
 138                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 139                           TALITOS1_CCCR_LO_RESET);
 140
 141                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
 142                         TALITOS1_CCCR_LO_RESET) && --timeout)
 143                         cpu_relax();
 144         } else {
 145                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 146                           TALITOS2_CCCR_RESET);
 147
 148                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 149                         TALITOS2_CCCR_RESET) && --timeout)
 150                         cpu_relax();
 151         }
 152
 153         if (timeout == 0) {
 154                 dev_err(dev, "failed to reset channel %d\n", ch);
 155                 return -EIO;
 156         }
 157
 158         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 159         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 160                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 161
 162         /* and ICCR writeback, if available */
 163         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 164                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 165                           TALITOS_CCCR_LO_IWSE);
 166
 167         return 0;
 168 }
 169
 170 static int reset_device(struct device *dev)
 171 {
 172         struct talitos_private *priv = dev_get_drvdata(dev);
 173         unsigned int timeout = TALITOS_TIMEOUT;
 174         bool is_sec1 = has_ftr_sec1(priv);
 175         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
 176
 177         setbits32(priv->reg + TALITOS_MCR, mcr);
 178
 179         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
 180                && --timeout)
 181                 cpu_relax();
 182
 183         if (priv->irq[1]) {
 184                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 185                 setbits32(priv->reg + TALITOS_MCR, mcr);
 186         }
 187
 188         if (timeout == 0) {
 189                 dev_err(dev, "failed to reset device\n");
 190                 return -EIO;
 191         }
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * Reset and initialize the device
 198  */
 199 static int init_device(struct device *dev)
 200 {
 201         struct talitos_private *priv = dev_get_drvdata(dev);
 202         int ch, err;
 203         bool is_sec1 = has_ftr_sec1(priv);
 204
 205         /*
 206          * Master reset
 207          * errata documentation: warning: certain SEC interrupts
 208          * are not fully cleared by writing the MCR:SWR bit,
 209          * set bit twice to completely reset
 210          */
 211         err = reset_device(dev);
 212         if (err)
 213                 return err;
 214
 215         err = reset_device(dev);
 216         if (err)
 217                 return err;
 218
 219         /* reset channels */
 220         for (ch = 0; ch < priv->num_channels; ch++) {
 221                 err = reset_channel(dev, ch);
 222                 if (err)
 223                         return err;
 224         }
 225
 226         /* enable channel done and error interrupts */
 227         if (is_sec1) {
 228                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
 229                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
 230                 /* disable parity error check in DEU (erroneous? test vect.) */
 231                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
 232         } else {
 233                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
 234                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
 235         }
 236
 237         /* disable integrity check error interrupts (use writeback instead) */
 238         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 239                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
 240                           TALITOS_MDEUICR_LO_ICE);
 241
 242         return 0;
 243 }
 244
 245 /**
 246  * talitos_submit - submits a descriptor to the device for processing
 247  * @dev:        the SEC device to be used
 248  * @ch:         the SEC device channel to be used
 249  * @desc:       the descriptor to be processed by the device
 250  * @callback:   whom to call when processing is complete
 251  * @context:    a handle for use by caller (optional)
 252  *
 253  * desc must contain valid dma-mapped (bus physical) address pointers.
 254  * callback must check err and feedback in descriptor header
 255  * for device processing status.
 256  */
 257 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 258                    void (*callback)(struct device *dev,
 259                                     struct talitos_desc *desc,
 260                                     void *context, int error),
 261                    void *context)
 262 {
 263         struct talitos_private *priv = dev_get_drvdata(dev);
 264         struct talitos_request *request;
 265         unsigned long flags;
 266         int head;
 267         bool is_sec1 = has_ftr_sec1(priv);
 268
 269         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 270
 271         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 272                 /* h/w fifo is full */
 273                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 274                 return -EAGAIN;
 275         }
 276
 277         head = priv->chan[ch].head;
 278         request = &priv->chan[ch].fifo[head];
 279
 280         /* map descriptor and save caller data */
 281         if (is_sec1) {
 282                 desc->hdr1 = desc->hdr;
 283                 desc->next_desc = 0;
 284                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
 285                                                    TALITOS_DESC_SIZE,
 286                                                    DMA_BIDIRECTIONAL);
 287         } else {
 288                 request->dma_desc = dma_map_single(dev, desc,
 289                                                    TALITOS_DESC_SIZE,
 290                                                    DMA_BIDIRECTIONAL);
 291         }
 292         request->callback = callback;
 293         request->context = context;
 294
 295         /* increment fifo head */
 296         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 297
 298         smp_wmb();
 299         request->desc = desc;
 300
 301         /* GO! */
 302         wmb();
 303         out_be32(priv->chan[ch].reg + TALITOS_FF,
 304                  upper_32_bits(request->dma_desc));
 305         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 306                  lower_32_bits(request->dma_desc));
 307
 308         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 309
 310         return -EINPROGRESS;
 311 }
 312 EXPORT_SYMBOL(talitos_submit);
 313
 314 /*
 315  * process what was done, notify callback of error if not
 316  */
 317 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 318 {
 319         struct talitos_private *priv = dev_get_drvdata(dev);
 320         struct talitos_request *request, saved_req;
 321         unsigned long flags;
 322         int tail, status;
 323         bool is_sec1 = has_ftr_sec1(priv);
 324
 325         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 326
 327         tail = priv->chan[ch].tail;
 328         while (priv->chan[ch].fifo[tail].desc) {
 329                 __be32 hdr;
 330
 331                 request = &priv->chan[ch].fifo[tail];
 332
 333                 /* descriptors with their done bits set don't get the error */
 334                 rmb();
 335                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
 336
 337                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 338                         status = 0;
 339                 else
 340                         if (!error)
 341                                 break;
 342                         else
 343                                 status = error;
 344
 345                 dma_unmap_single(dev, request->dma_desc,
 346                                  TALITOS_DESC_SIZE,
 347                                  DMA_BIDIRECTIONAL);
 348
 349                 /* copy entries so we can call callback outside lock */
 350                 saved_req.desc = request->desc;
 351                 saved_req.callback = request->callback;
 352                 saved_req.context = request->context;
 353
 354                 /* release request entry in fifo */
 355                 smp_wmb();
 356                 request->desc = NULL;
 357
 358                 /* increment fifo tail */
 359                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 360
 361                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 362
 363                 atomic_dec(&priv->chan[ch].submit_count);
 364
 365                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 366                                    status);
 367                 /* channel may resume processing in single desc error case */
 368                 if (error && !reset_ch && status == error)
 369                         return;
 370                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 371                 tail = priv->chan[ch].tail;
 372         }
 373
 374         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 375 }
 376
 377 /*
 378  * process completed requests for channels that have done status
 379  */
 380 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
 381 static void talitos1_done_##name(unsigned long data)                    \
 382 {                                                                       \
 383         struct device *dev = (struct device *)data;                     \
 384         struct talitos_private *priv = dev_get_drvdata(dev);            \
 385         unsigned long flags;                                            \
 386                                                                         \
 387         if (ch_done_mask & 0x10000000)                                  \
 388                 flush_channel(dev, 0, 0, 0);                    \
 389         if (priv->num_channels == 1)                                    \
 390                 goto out;                                               \
 391         if (ch_done_mask & 0x40000000)                                  \
 392                 flush_channel(dev, 1, 0, 0);                    \
 393         if (ch_done_mask & 0x00010000)                                  \
 394                 flush_channel(dev, 2, 0, 0);                    \
 395         if (ch_done_mask & 0x00040000)                                  \
 396                 flush_channel(dev, 3, 0, 0);                    \
 397                                                                         \
 398 out:                                                                    \
 399         /* At this point, all completed channels have been processed */ \
 400         /* Unmask done interrupts for channels completed later on. */   \
 401         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 402         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 403         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
 404         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 405 }
 406
 407 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
 408
 409 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
 410 static void talitos2_done_##name(unsigned long data)                    \
 411 {                                                                       \
 412         struct device *dev = (struct device *)data;                     \
 413         struct talitos_private *priv = dev_get_drvdata(dev);            \
 414         unsigned long flags;                                            \
 415                                                                         \
 416         if (ch_done_mask & 1)                                           \
 417                 flush_channel(dev, 0, 0, 0);                            \
 418         if (priv->num_channels == 1)                                    \
 419                 goto out;                                               \
 420         if (ch_done_mask & (1 << 2))                                    \
 421                 flush_channel(dev, 1, 0, 0);                            \
 422         if (ch_done_mask & (1 << 4))                                    \
 423                 flush_channel(dev, 2, 0, 0);                            \
 424         if (ch_done_mask & (1 << 6))                                    \
 425                 flush_channel(dev, 3, 0, 0);                            \
 426                                                                         \
 427 out:                                                                    \
 428         /* At this point, all completed channels have been processed */ \
 429         /* Unmask done interrupts for channels completed later on. */   \
 430         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 431         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 432         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
 433         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 434 }
 435
 436 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
 437 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
 438 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
 439
 440 /*
 441  * locate current (offending) descriptor
 442  */
 443 static u32 current_desc_hdr(struct device *dev, int ch)
 444 {
 445         struct talitos_private *priv = dev_get_drvdata(dev);
 446         int tail, iter;
 447         dma_addr_t cur_desc;
 448
 449         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 450         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 451
 452         if (!cur_desc) {
 453                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
 454                 return 0;
 455         }
 456
 457         tail = priv->chan[ch].tail;
 458
 459         iter = tail;
 460         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
 461                 iter = (iter + 1) & (priv->fifo_len - 1);
 462                 if (iter == tail) {
 463                         dev_err(dev, "couldn't locate current descriptor\n");
 464                         return 0;
 465                 }
 466         }
 467
 468         return priv->chan[ch].fifo[iter].desc->hdr;
 469 }
 470
 471 /*
 472  * user diagnostics; report root cause of error based on execution unit status
 473  */
 474 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 475 {
 476         struct talitos_private *priv = dev_get_drvdata(dev);
 477         int i;
 478
 479         if (!desc_hdr)
 480                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
 481
 482         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 483         case DESC_HDR_SEL0_AFEU:
 484                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 485                         in_be32(priv->reg_afeu + TALITOS_EUISR),
 486                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
 487                 break;
 488         case DESC_HDR_SEL0_DEU:
 489                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 490                         in_be32(priv->reg_deu + TALITOS_EUISR),
 491                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
 492                 break;
 493         case DESC_HDR_SEL0_MDEUA:
 494         case DESC_HDR_SEL0_MDEUB:
 495                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 496                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 497                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 498                 break;
 499         case DESC_HDR_SEL0_RNG:
 500                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 501                         in_be32(priv->reg_rngu + TALITOS_ISR),
 502                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
 503                 break;
 504         case DESC_HDR_SEL0_PKEU:
 505                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 506                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 507                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 508                 break;
 509         case DESC_HDR_SEL0_AESU:
 510                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 511                         in_be32(priv->reg_aesu + TALITOS_EUISR),
 512                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
 513                 break;
 514         case DESC_HDR_SEL0_CRCU:
 515                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 516                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 517                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 518                 break;
 519         case DESC_HDR_SEL0_KEU:
 520                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 521                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 522                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 523                 break;
 524         }
 525
 526         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 527         case DESC_HDR_SEL1_MDEUA:
 528         case DESC_HDR_SEL1_MDEUB:
 529                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 530                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 531                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 532                 break;
 533         case DESC_HDR_SEL1_CRCU:
 534                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 535                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 536                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 537                 break;
 538         }
 539
 540         for (i = 0; i < 8; i++)
 541                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 542                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 543                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 544 }
 545
 546 /*
 547  * recover from error interrupts
 548  */
 549 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 550 {
 551         struct talitos_private *priv = dev_get_drvdata(dev);
 552         unsigned int timeout = TALITOS_TIMEOUT;
 553         int ch, error, reset_dev = 0;
 554         u32 v_lo;
 555         bool is_sec1 = has_ftr_sec1(priv);
 556         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
 557
 558         for (ch = 0; ch < priv->num_channels; ch++) {
 559                 /* skip channels without errors */
 560                 if (is_sec1) {
 561                         /* bits 29, 31, 17, 19 */
 562                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
 563                                 continue;
 564                 } else {
 565                         if (!(isr & (1 << (ch * 2 + 1))))
 566                                 continue;
 567                 }
 568
 569                 error = -EINVAL;
 570
 571                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 572
 573                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 574                         dev_err(dev, "double fetch fifo overflow error\n");
 575                         error = -EAGAIN;
 576                         reset_ch = 1;
 577                 }
 578                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 579                         /* h/w dropped descriptor */
 580                         dev_err(dev, "single fetch fifo overflow error\n");
 581                         error = -EAGAIN;
 582                 }
 583                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 584                         dev_err(dev, "master data transfer error\n");
 585                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 586                         dev_err(dev, is_sec1 ? "pointeur not complete error\n"
 587                                              : "s/g data length zero error\n");
 588                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 589                         dev_err(dev, is_sec1 ? "parity error\n"
 590                                              : "fetch pointer zero error\n");
 591                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 592                         dev_err(dev, "illegal descriptor header error\n");
 593                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 594                         dev_err(dev, is_sec1 ? "static assignment error\n"
 595                                              : "invalid exec unit error\n");
 596                 if (v_lo & TALITOS_CCPSR_LO_EU)
 597                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 598                 if (!is_sec1) {
 599                         if (v_lo & TALITOS_CCPSR_LO_GB)
 600                                 dev_err(dev, "gather boundary error\n");
 601                         if (v_lo & TALITOS_CCPSR_LO_GRL)
 602                                 dev_err(dev, "gather return/length error\n");
 603                         if (v_lo & TALITOS_CCPSR_LO_SB)
 604                                 dev_err(dev, "scatter boundary error\n");
 605                         if (v_lo & TALITOS_CCPSR_LO_SRL)
 606                                 dev_err(dev, "scatter return/length error\n");
 607                 }
 608
 609                 flush_channel(dev, ch, error, reset_ch);
 610
 611                 if (reset_ch) {
 612                         reset_channel(dev, ch);
 613                 } else {
 614                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 615                                   TALITOS2_CCCR_CONT);
 616                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 617                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 618                                TALITOS2_CCCR_CONT) && --timeout)
 619                                 cpu_relax();
 620                         if (timeout == 0) {
 621                                 dev_err(dev, "failed to restart channel %d\n",
 622                                         ch);
 623                                 reset_dev = 1;
 624                         }
 625                 }
 626         }
 627         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
 628             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
 629                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
 630                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
 631                                 isr, isr_lo);
 632                 else
 633                         dev_err(dev, "done overflow, internal time out, or "
 634                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
 635
 636                 /* purge request queues */
 637                 for (ch = 0; ch < priv->num_channels; ch++)
 638                         flush_channel(dev, ch, -EIO, 1);
 639
 640                 /* reset and reinitialize the device */
 641                 init_device(dev);
 642         }
 643 }
 644
 645 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 646 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
 647 {                                                                              \
 648         struct device *dev = data;                                             \
 649         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 650         u32 isr, isr_lo;                                                       \
 651         unsigned long flags;                                                   \
 652                                                                                \
 653         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 654         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 655         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 656         /* Acknowledge interrupt */                                            \
 657         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 658         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 659                                                                                \
 660         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
 661                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 662                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 663         }                                                                      \
 664         else {                                                                 \
 665                 if (likely(isr & ch_done_mask)) {                              \
 666                         /* mask further done interrupts. */                    \
 667                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 668                         /* done_task will unmask done interrupts at exit */    \
 669                         tasklet_schedule(&priv->done_task[tlet]);              \
 670                 }                                                              \
 671                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 672         }                                                                      \
 673                                                                                \
 674         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 675                                                                 IRQ_NONE;      \
 676 }
 677
 678 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
 679
 680 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 681 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
 682 {                                                                              \
 683         struct device *dev = data;                                             \
 684         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 685         u32 isr, isr_lo;                                                       \
 686         unsigned long flags;                                                   \
 687                                                                                \
 688         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 689         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 690         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 691         /* Acknowledge interrupt */                                            \
 692         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 693         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 694                                                                                \
 695         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
 696                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 697                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 698         }                                                                      \
 699         else {                                                                 \
 700                 if (likely(isr & ch_done_mask)) {                              \
 701                         /* mask further done interrupts. */                    \
 702                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 703                         /* done_task will unmask done interrupts at exit */    \
 704                         tasklet_schedule(&priv->done_task[tlet]);              \
 705                 }                                                              \
 706                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 707         }                                                                      \
 708                                                                                \
 709         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 710                                                                 IRQ_NONE;      \
 711 }
 712
 713 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
 714 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
 715                        0)
 716 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
 717                        1)
 718
 719 /*
 720  * hwrng
 721  */
 722 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 723 {
 724         struct device *dev = (struct device *)rng->priv;
 725         struct talitos_private *priv = dev_get_drvdata(dev);
 726         u32 ofl;
 727         int i;
 728
 729         for (i = 0; i < 20; i++) {
 730                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
 731                       TALITOS_RNGUSR_LO_OFL;
 732                 if (ofl || !wait)
 733                         break;
 734                 udelay(10);
 735         }
 736
 737         return !!ofl;
 738 }
 739
 740 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 741 {
 742         struct device *dev = (struct device *)rng->priv;
 743         struct talitos_private *priv = dev_get_drvdata(dev);
 744
 745         /* rng fifo requires 64-bit accesses */
 746         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
 747         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
 748
 749         return sizeof(u32);
 750 }
 751
 752 static int talitos_rng_init(struct hwrng *rng)
 753 {
 754         struct device *dev = (struct device *)rng->priv;
 755         struct talitos_private *priv = dev_get_drvdata(dev);
 756         unsigned int timeout = TALITOS_TIMEOUT;
 757
 758         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
 759         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
 760                  & TALITOS_RNGUSR_LO_RD)
 761                && --timeout)
 762                 cpu_relax();
 763         if (timeout == 0) {
 764                 dev_err(dev, "failed to reset rng hw\n");
 765                 return -ENODEV;
 766         }
 767
 768         /* start generating */
 769         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
 770
 771         return 0;
 772 }
 773
 774 static int talitos_register_rng(struct device *dev)
 775 {
 776         struct talitos_private *priv = dev_get_drvdata(dev);
 777         int err;
 778
 779         priv->rng.name          = dev_driver_string(dev),
 780         priv->rng.init          = talitos_rng_init,
 781         priv->rng.data_present  = talitos_rng_data_present,
 782         priv->rng.data_read     = talitos_rng_data_read,
 783         priv->rng.priv          = (unsigned long)dev;
 784
 785         err = hwrng_register(&priv->rng);
 786         if (!err)
 787                 priv->rng_registered = true;
 788
 789         return err;
 790 }
 791
 792 static void talitos_unregister_rng(struct device *dev)
 793 {
 794         struct talitos_private *priv = dev_get_drvdata(dev);
 795
 796         if (!priv->rng_registered)
 797                 return;
 798
 799         hwrng_unregister(&priv->rng);
 800         priv->rng_registered = false;
 801 }
 802
 803 /*
 804  * crypto alg
 805  */
 806 #define TALITOS_CRA_PRIORITY            3000
 807 #define TALITOS_MAX_KEY_SIZE            (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
 808 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 809
 810 struct talitos_ctx {
 811         struct device *dev;
 812         int ch;
 813         __be32 desc_hdr_template;
 814         u8 key[TALITOS_MAX_KEY_SIZE];
 815         u8 iv[TALITOS_MAX_IV_LENGTH];
 816         unsigned int keylen;
 817         unsigned int enckeylen;
 818         unsigned int authkeylen;
 819 };
 820
 821 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 822 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 823
 824 struct talitos_ahash_req_ctx {
 825         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 826         unsigned int hw_context_size;
 827         u8 buf[HASH_MAX_BLOCK_SIZE];
 828         u8 bufnext[HASH_MAX_BLOCK_SIZE];
 829         unsigned int swinit;
 830         unsigned int first;
 831         unsigned int last;
 832         unsigned int to_hash_later;
 833         unsigned int nbuf;
 834         struct scatterlist bufsl[2];
 835         struct scatterlist *psrc;
 836 };
 837
 838 struct talitos_export_state {
 839         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 840         u8 buf[HASH_MAX_BLOCK_SIZE];
 841         unsigned int swinit;
 842         unsigned int first;
 843         unsigned int last;
 844         unsigned int to_hash_later;
 845         unsigned int nbuf;
 846 };
 847
 848 static int aead_setkey(struct crypto_aead *authenc,
 849                        const u8 *key, unsigned int keylen)
 850 {
 851         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 852         struct crypto_authenc_keys keys;
 853
 854         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 855                 goto badkey;
 856
 857         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 858                 goto badkey;
 859
 860         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 861         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 862
 863         ctx->keylen = keys.authkeylen + keys.enckeylen;
 864         ctx->enckeylen = keys.enckeylen;
 865         ctx->authkeylen = keys.authkeylen;
 866
 867         return 0;
 868
 869 badkey:
 870         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 871         return -EINVAL;
 872 }
 873
 874 /*
 875  * talitos_edesc - s/w-extended descriptor
 876  * @src_nents: number of segments in input scatterlist
 877  * @dst_nents: number of segments in output scatterlist
 878  * @icv_ool: whether ICV is out-of-line
 879  * @iv_dma: dma address of iv for checking continuity and link table
 880  * @dma_len: length of dma mapped link_tbl space
 881  * @dma_link_tbl: bus physical address of link_tbl/buf
 882  * @desc: h/w descriptor
 883  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
 884  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
 885  *
 886  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 887  * is greater than 1, an integrity check value is concatenated to the end
 888  * of link_tbl data
 889  */
 890 struct talitos_edesc {
 891         int src_nents;
 892         int dst_nents;
 893         bool icv_ool;
 894         dma_addr_t iv_dma;
 895         int dma_len;
 896         dma_addr_t dma_link_tbl;
 897         struct talitos_desc desc;
 898         union {
 899                 struct talitos_ptr link_tbl[0];
 900                 u8 buf[0];
 901         };
 902 };
 903
 904 static void talitos_sg_unmap(struct device *dev,
 905                              struct talitos_edesc *edesc,
 906                              struct scatterlist *src,
 907                              struct scatterlist *dst)
 908 {
 909         unsigned int src_nents = edesc->src_nents ? : 1;
 910         unsigned int dst_nents = edesc->dst_nents ? : 1;
 911
 912         if (src != dst) {
 913                 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 914
 915                 if (dst) {
 916                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 917                 }
 918         } else
 919                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 920 }
 921
 922 static void ipsec_esp_unmap(struct device *dev,
 923                             struct talitos_edesc *edesc,
 924                             struct aead_request *areq)
 925 {
 926         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 927         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 928         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 929         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 930
 931         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 932
 933         if (edesc->dma_len)
 934                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 935                                  DMA_BIDIRECTIONAL);
 936 }
 937
 938 /*
 939  * ipsec_esp descriptor callbacks
 940  */
 941 static void ipsec_esp_encrypt_done(struct device *dev,
 942                                    struct talitos_desc *desc, void *context,
 943                                    int err)
 944 {
 945         struct aead_request *areq = context;
 946         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 947         unsigned int authsize = crypto_aead_authsize(authenc);
 948         struct talitos_edesc *edesc;
 949         struct scatterlist *sg;
 950         void *icvdata;
 951
 952         edesc = container_of(desc, struct talitos_edesc, desc);
 953
 954         ipsec_esp_unmap(dev, edesc, areq);
 955
 956         /* copy the generated ICV to dst */
 957         if (edesc->icv_ool) {
 958                 icvdata = &edesc->link_tbl[edesc->src_nents +
 959                                            edesc->dst_nents + 2];
 960                 sg = sg_last(areq->dst, edesc->dst_nents);
 961                 memcpy((char *)sg_virt(sg) + sg->length - authsize,
 962                        icvdata, authsize);
 963         }
 964
 965         kfree(edesc);
 966
 967         aead_request_complete(areq, err);
 968 }
 969
 970 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
 971                                           struct talitos_desc *desc,
 972                                           void *context, int err)
 973 {
 974         struct aead_request *req = context;
 975         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 976         unsigned int authsize = crypto_aead_authsize(authenc);
 977         struct talitos_edesc *edesc;
 978         struct scatterlist *sg;
 979         char *oicv, *icv;
 980
 981         edesc = container_of(desc, struct talitos_edesc, desc);
 982
 983         ipsec_esp_unmap(dev, edesc, req);
 984
 985         if (!err) {
 986                 /* auth check */
 987                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
 988                 icv = (char *)sg_virt(sg) + sg->length - authsize;
 989
 990                 if (edesc->dma_len) {
 991                         oicv = (char *)&edesc->link_tbl[edesc->src_nents +
 992                                                         edesc->dst_nents + 2];
 993                         if (edesc->icv_ool)
 994                                 icv = oicv + authsize;
 995                 } else
 996                         oicv = (char *)&edesc->link_tbl[0];
 997
 998                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
 999         }
1000
1001         kfree(edesc);
1002
1003         aead_request_complete(req, err);
1004 }
1005
1006 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1007                                           struct talitos_desc *desc,
1008                                           void *context, int err)
1009 {
1010         struct aead_request *req = context;
1011         struct talitos_edesc *edesc;
1012
1013         edesc = container_of(desc, struct talitos_edesc, desc);
1014
1015         ipsec_esp_unmap(dev, edesc, req);
1016
1017         /* check ICV auth status */
1018         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1019                      DESC_HDR_LO_ICCR1_PASS))
1020                 err = -EBADMSG;
1021
1022         kfree(edesc);
1023
1024         aead_request_complete(req, err);
1025 }
1026
1027 /*
1028  * convert scatterlist to SEC h/w link table format
1029  * stop at cryptlen bytes
1030  */
1031 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1032                                  unsigned int offset, int cryptlen,
1033                                  struct talitos_ptr *link_tbl_ptr)
1034 {
1035         int n_sg = sg_count;
1036         int count = 0;
1037
1038         while (cryptlen && sg && n_sg--) {
1039                 unsigned int len = sg_dma_len(sg);
1040
1041                 if (offset >= len) {
1042                         offset -= len;
1043                         goto next;
1044                 }
1045
1046                 len -= offset;
1047
1048                 if (len > cryptlen)
1049                         len = cryptlen;
1050
1051                 to_talitos_ptr(link_tbl_ptr + count,
1052                                sg_dma_address(sg) + offset, 0);
1053                 link_tbl_ptr[count].len = cpu_to_be16(len);
1054                 link_tbl_ptr[count].j_extent = 0;
1055                 count++;
1056                 cryptlen -= len;
1057                 offset = 0;
1058
1059 next:
1060                 sg = sg_next(sg);
1061         }
1062
1063         /* tag end of link table */
1064         if (count > 0)
1065                 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN;
1066
1067         return count;
1068 }
1069
1070 static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1071                                  int cryptlen,
1072                                  struct talitos_ptr *link_tbl_ptr)
1073 {
1074         return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen,
1075                                      link_tbl_ptr);
1076 }
1077
1078 /*
1079  * fill in and submit ipsec_esp descriptor
1080  */
1081 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1082                      void (*callback)(struct device *dev,
1083                                       struct talitos_desc *desc,
1084                                       void *context, int error))
1085 {
1086         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1087         unsigned int authsize = crypto_aead_authsize(aead);
1088         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1089         struct device *dev = ctx->dev;
1090         struct talitos_desc *desc = &edesc->desc;
1091         unsigned int cryptlen = areq->cryptlen;
1092         unsigned int ivsize = crypto_aead_ivsize(aead);
1093         int tbl_off = 0;
1094         int sg_count, ret;
1095         int sg_link_tbl_len;
1096
1097         /* hmac key */
1098         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1099                                DMA_TO_DEVICE);
1100
1101         sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1,
1102                               (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1103                                                            : DMA_TO_DEVICE);
1104         /* hmac data */
1105         desc->ptr[1].len = cpu_to_be16(areq->assoclen);
1106         if (sg_count > 1 &&
1107             (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0,
1108                                          areq->assoclen,
1109                                          &edesc->link_tbl[tbl_off])) > 1) {
1110                 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1111                                sizeof(struct talitos_ptr), 0);
1112                 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1113
1114                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1115                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1116
1117                 tbl_off += ret;
1118         } else {
1119                 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0);
1120                 desc->ptr[1].j_extent = 0;
1121         }
1122
1123         /* cipher iv */
1124         to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1125         desc->ptr[2].len = cpu_to_be16(ivsize);
1126         desc->ptr[2].j_extent = 0;
1127
1128         /* cipher key */
1129         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1130                                (char *)&ctx->key + ctx->authkeylen,
1131                                DMA_TO_DEVICE);
1132
1133         /*
1134          * cipher in
1135          * map and adjust cipher len to aead request cryptlen.
1136          * extent is bytes of HMAC postpended to ciphertext,
1137          * typically 12 for ipsec
1138          */
1139         desc->ptr[4].len = cpu_to_be16(cryptlen);
1140         desc->ptr[4].j_extent = authsize;
1141
1142         sg_link_tbl_len = cryptlen;
1143         if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1144                 sg_link_tbl_len += authsize;
1145
1146         if (sg_count == 1) {
1147                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) +
1148                                areq->assoclen, 0);
1149         } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count,
1150                                                 areq->assoclen, sg_link_tbl_len,
1151                                                 &edesc->link_tbl[tbl_off])) >
1152                    1) {
1153                 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1154                 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1155                                               tbl_off *
1156                                               sizeof(struct talitos_ptr), 0);
1157                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1158                                            edesc->dma_len,
1159                                            DMA_BIDIRECTIONAL);
1160                 tbl_off += ret;
1161         } else {
1162                 copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0);
1163         }
1164
1165         /* cipher out */
1166         desc->ptr[5].len = cpu_to_be16(cryptlen);
1167         desc->ptr[5].j_extent = authsize;
1168
1169         if (areq->src != areq->dst)
1170                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
1171                                       DMA_FROM_DEVICE);
1172
1173         edesc->icv_ool = false;
1174
1175         if (sg_count == 1) {
1176                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) +
1177                                areq->assoclen, 0);
1178         } else if ((sg_count =
1179                         sg_to_link_tbl_offset(areq->dst, sg_count,
1180                                               areq->assoclen, cryptlen,
1181                                               &edesc->link_tbl[tbl_off])) > 1) {
1182                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1183
1184                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1185                                tbl_off * sizeof(struct talitos_ptr), 0);
1186
1187                 /* Add an entry to the link table for ICV data */
1188                 tbl_ptr += sg_count - 1;
1189                 tbl_ptr->j_extent = 0;
1190                 tbl_ptr++;
1191                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1192                 tbl_ptr->len = cpu_to_be16(authsize);
1193
1194                 /* icv data follows link tables */
1195                 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1196                                         (edesc->src_nents + edesc->dst_nents +
1197                                          2) * sizeof(struct talitos_ptr) +
1198                                         authsize, 0);
1199                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1200                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1201                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1202
1203                 edesc->icv_ool = true;
1204         } else {
1205                 copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0);
1206         }
1207
1208         /* iv out */
1209         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1210                                DMA_FROM_DEVICE);
1211
1212         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1213         if (ret != -EINPROGRESS) {
1214                 ipsec_esp_unmap(dev, edesc, areq);
1215                 kfree(edesc);
1216         }
1217         return ret;
1218 }
1219
1220 /*
1221  * allocate and map the extended descriptor
1222  */
1223 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1224                                                  struct scatterlist *src,
1225                                                  struct scatterlist *dst,
1226                                                  u8 *iv,
1227                                                  unsigned int assoclen,
1228                                                  unsigned int cryptlen,
1229                                                  unsigned int authsize,
1230                                                  unsigned int ivsize,
1231                                                  int icv_stashing,
1232                                                  u32 cryptoflags,
1233                                                  bool encrypt)
1234 {
1235         struct talitos_edesc *edesc;
1236         int src_nents, dst_nents, alloc_len, dma_len;
1237         dma_addr_t iv_dma = 0;
1238         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1239                       GFP_ATOMIC;
1240         struct talitos_private *priv = dev_get_drvdata(dev);
1241         bool is_sec1 = has_ftr_sec1(priv);
1242         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1243
1244         if (cryptlen + authsize > max_len) {
1245                 dev_err(dev, "length exceeds h/w max limit\n");
1246                 return ERR_PTR(-EINVAL);
1247         }
1248
1249         if (ivsize)
1250                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1251
1252         if (!dst || dst == src) {
1253                 src_nents = sg_nents_for_len(src,
1254                                              assoclen + cryptlen + authsize);
1255                 src_nents = (src_nents == 1) ? 0 : src_nents;
1256                 dst_nents = dst ? src_nents : 0;
1257         } else { /* dst && dst != src*/
1258                 src_nents = sg_nents_for_len(src, assoclen + cryptlen +
1259                                                  (encrypt ? 0 : authsize));
1260                 src_nents = (src_nents == 1) ? 0 : src_nents;
1261                 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen +
1262                                                  (encrypt ? authsize : 0));
1263                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1264         }
1265
1266         /*
1267          * allocate space for base edesc plus the link tables,
1268          * allowing for two separate entries for AD and generated ICV (+ 2),
1269          * and space for two sets of ICVs (stashed and generated)
1270          */
1271         alloc_len = sizeof(struct talitos_edesc);
1272         if (src_nents || dst_nents) {
1273                 if (is_sec1)
1274                         dma_len = (src_nents ? cryptlen : 0) +
1275                                   (dst_nents ? cryptlen : 0);
1276                 else
1277                         dma_len = (src_nents + dst_nents + 2) *
1278                                   sizeof(struct talitos_ptr) + authsize * 2;
1279                 alloc_len += dma_len;
1280         } else {
1281                 dma_len = 0;
1282                 alloc_len += icv_stashing ? authsize : 0;
1283         }
1284
1285         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1286         if (!edesc) {
1287                 if (iv_dma)
1288                         dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1289
1290                 dev_err(dev, "could not allocate edescriptor\n");
1291                 return ERR_PTR(-ENOMEM);
1292         }
1293
1294         edesc->src_nents = src_nents;
1295         edesc->dst_nents = dst_nents;
1296         edesc->iv_dma = iv_dma;
1297         edesc->dma_len = dma_len;
1298         if (dma_len)
1299                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1300                                                      edesc->dma_len,
1301                                                      DMA_BIDIRECTIONAL);
1302
1303         return edesc;
1304 }
1305
1306 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1307                                               int icv_stashing, bool encrypt)
1308 {
1309         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1310         unsigned int authsize = crypto_aead_authsize(authenc);
1311         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1312         unsigned int ivsize = crypto_aead_ivsize(authenc);
1313
1314         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1315                                    iv, areq->assoclen, areq->cryptlen,
1316                                    authsize, ivsize, icv_stashing,
1317                                    areq->base.flags, encrypt);
1318 }
1319
1320 static int aead_encrypt(struct aead_request *req)
1321 {
1322         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1323         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1324         struct talitos_edesc *edesc;
1325
1326         /* allocate extended descriptor */
1327         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1328         if (IS_ERR(edesc))
1329                 return PTR_ERR(edesc);
1330
1331         /* set encrypt */
1332         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1333
1334         return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1335 }
1336
1337 static int aead_decrypt(struct aead_request *req)
1338 {
1339         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1340         unsigned int authsize = crypto_aead_authsize(authenc);
1341         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1342         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1343         struct talitos_edesc *edesc;
1344         struct scatterlist *sg;
1345         void *icvdata;
1346
1347         req->cryptlen -= authsize;
1348
1349         /* allocate extended descriptor */
1350         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1351         if (IS_ERR(edesc))
1352                 return PTR_ERR(edesc);
1353
1354         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1355             ((!edesc->src_nents && !edesc->dst_nents) ||
1356              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1357
1358                 /* decrypt and check the ICV */
1359                 edesc->desc.hdr = ctx->desc_hdr_template |
1360                                   DESC_HDR_DIR_INBOUND |
1361                                   DESC_HDR_MODE1_MDEU_CICV;
1362
1363                 /* reset integrity check result bits */
1364                 edesc->desc.hdr_lo = 0;
1365
1366                 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1367         }
1368
1369         /* Have to check the ICV with software */
1370         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1371
1372         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1373         if (edesc->dma_len)
1374                 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1375                                                    edesc->dst_nents + 2];
1376         else
1377                 icvdata = &edesc->link_tbl[0];
1378
1379         sg = sg_last(req->src, edesc->src_nents ? : 1);
1380
1381         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1382
1383         return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1384 }
1385
1386 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1387                              const u8 *key, unsigned int keylen)
1388 {
1389         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1390
1391         if (keylen > TALITOS_MAX_KEY_SIZE) {
1392                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1393                 return -EINVAL;
1394         }
1395
1396         memcpy(&ctx->key, key, keylen);
1397         ctx->keylen = keylen;
1398
1399         return 0;
1400 }
1401
1402 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1403                                  struct scatterlist *dst, unsigned int len,
1404                                  struct talitos_edesc *edesc)
1405 {
1406         struct talitos_private *priv = dev_get_drvdata(dev);
1407         bool is_sec1 = has_ftr_sec1(priv);
1408
1409         if (is_sec1) {
1410                 if (!edesc->src_nents) {
1411                         dma_unmap_sg(dev, src, 1,
1412                                      dst != src ? DMA_TO_DEVICE
1413                                                 : DMA_BIDIRECTIONAL);
1414                 }
1415                 if (dst && edesc->dst_nents) {
1416                         dma_sync_single_for_device(dev,
1417                                                    edesc->dma_link_tbl + len,
1418                                                    len, DMA_FROM_DEVICE);
1419                         sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1420                                             edesc->buf + len, len);
1421                 } else if (dst && dst != src) {
1422                         dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1423                 }
1424         } else {
1425                 talitos_sg_unmap(dev, edesc, src, dst);
1426         }
1427 }
1428
1429 static void common_nonsnoop_unmap(struct device *dev,
1430                                   struct talitos_edesc *edesc,
1431                                   struct ablkcipher_request *areq)
1432 {
1433         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1434
1435         unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1436         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1437         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1438
1439         if (edesc->dma_len)
1440                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1441                                  DMA_BIDIRECTIONAL);
1442 }
1443
1444 static void ablkcipher_done(struct device *dev,
1445                             struct talitos_desc *desc, void *context,
1446                             int err)
1447 {
1448         struct ablkcipher_request *areq = context;
1449         struct talitos_edesc *edesc;
1450
1451         edesc = container_of(desc, struct talitos_edesc, desc);
1452
1453         common_nonsnoop_unmap(dev, edesc, areq);
1454
1455         kfree(edesc);
1456
1457         areq->base.complete(&areq->base, err);
1458 }
1459
1460 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1461                           unsigned int len, struct talitos_edesc *edesc,
1462                           enum dma_data_direction dir, struct talitos_ptr *ptr)
1463 {
1464         int sg_count;
1465         struct talitos_private *priv = dev_get_drvdata(dev);
1466         bool is_sec1 = has_ftr_sec1(priv);
1467
1468         to_talitos_ptr_len(ptr, len, is_sec1);
1469
1470         if (is_sec1) {
1471                 sg_count = edesc->src_nents ? : 1;
1472
1473                 if (sg_count == 1) {
1474                         dma_map_sg(dev, src, 1, dir);
1475                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1476                 } else {
1477                         sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1478                         to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1479                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1480                                                    len, DMA_TO_DEVICE);
1481                 }
1482         } else {
1483                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1484
1485                 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir);
1486
1487                 if (sg_count == 1) {
1488                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1489                 } else {
1490                         sg_count = sg_to_link_tbl(src, sg_count, len,
1491                                                   &edesc->link_tbl[0]);
1492                         if (sg_count > 1) {
1493                                 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1494                                 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1495                                 dma_sync_single_for_device(dev,
1496                                                            edesc->dma_link_tbl,
1497                                                            edesc->dma_len,
1498                                                            DMA_BIDIRECTIONAL);
1499                         } else {
1500                                 /* Only one segment now, so no link tbl needed*/
1501                                 to_talitos_ptr(ptr, sg_dma_address(src),
1502                                                is_sec1);
1503                         }
1504                 }
1505         }
1506         return sg_count;
1507 }
1508
1509 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1510                             unsigned int len, struct talitos_edesc *edesc,
1511                             enum dma_data_direction dir,
1512                             struct talitos_ptr *ptr, int sg_count)
1513 {
1514         struct talitos_private *priv = dev_get_drvdata(dev);
1515         bool is_sec1 = has_ftr_sec1(priv);
1516
1517         if (dir != DMA_NONE)
1518                 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir);
1519
1520         to_talitos_ptr_len(ptr, len, is_sec1);
1521
1522         if (is_sec1) {
1523                 if (sg_count == 1) {
1524                         if (dir != DMA_NONE)
1525                                 dma_map_sg(dev, dst, 1, dir);
1526                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1527                 } else {
1528                         to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1529                         dma_sync_single_for_device(dev,
1530                                                    edesc->dma_link_tbl + len,
1531                                                    len, DMA_FROM_DEVICE);
1532                 }
1533         } else {
1534                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1535
1536                 if (sg_count == 1) {
1537                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1538                 } else {
1539                         struct talitos_ptr *link_tbl_ptr =
1540                                 &edesc->link_tbl[edesc->src_nents + 1];
1541
1542                         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1543                                             (edesc->src_nents + 1) *
1544                                              sizeof(struct talitos_ptr), 0);
1545                         ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1546                         sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1547                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1548                                                    edesc->dma_len,
1549                                                    DMA_BIDIRECTIONAL);
1550                 }
1551         }
1552 }
1553
1554 static int common_nonsnoop(struct talitos_edesc *edesc,
1555                            struct ablkcipher_request *areq,
1556                            void (*callback) (struct device *dev,
1557                                              struct talitos_desc *desc,
1558                                              void *context, int error))
1559 {
1560         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1561         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1562         struct device *dev = ctx->dev;
1563         struct talitos_desc *desc = &edesc->desc;
1564         unsigned int cryptlen = areq->nbytes;
1565         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1566         int sg_count, ret;
1567         struct talitos_private *priv = dev_get_drvdata(dev);
1568         bool is_sec1 = has_ftr_sec1(priv);
1569
1570         /* first DWORD empty */
1571         desc->ptr[0] = zero_entry;
1572
1573         /* cipher iv */
1574         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1575         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1576         to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1577
1578         /* cipher key */
1579         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1580                                (char *)&ctx->key, DMA_TO_DEVICE);
1581
1582         /*
1583          * cipher in
1584          */
1585         sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1586                                          (areq->src == areq->dst) ?
1587                                           DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1588                                           &desc->ptr[3]);
1589
1590         /* cipher out */
1591         map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1592                                (areq->src == areq->dst) ? DMA_NONE
1593                                                         : DMA_FROM_DEVICE,
1594                                &desc->ptr[4], sg_count);
1595
1596         /* iv out */
1597         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1598                                DMA_FROM_DEVICE);
1599
1600         /* last DWORD empty */
1601         desc->ptr[6] = zero_entry;
1602
1603         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1604         if (ret != -EINPROGRESS) {
1605                 common_nonsnoop_unmap(dev, edesc, areq);
1606                 kfree(edesc);
1607         }
1608         return ret;
1609 }
1610
1611 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1612                                                     areq, bool encrypt)
1613 {
1614         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1615         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1616         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1617
1618         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1619                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1620                                    areq->base.flags, encrypt);
1621 }
1622
1623 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1624 {
1625         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1626         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1627         struct talitos_edesc *edesc;
1628
1629         /* allocate extended descriptor */
1630         edesc = ablkcipher_edesc_alloc(areq, true);
1631         if (IS_ERR(edesc))
1632                 return PTR_ERR(edesc);
1633
1634         /* set encrypt */
1635         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1636
1637         return common_nonsnoop(edesc, areq, ablkcipher_done);
1638 }
1639
1640 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1641 {
1642         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1643         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1644         struct talitos_edesc *edesc;
1645
1646         /* allocate extended descriptor */
1647         edesc = ablkcipher_edesc_alloc(areq, false);
1648         if (IS_ERR(edesc))
1649                 return PTR_ERR(edesc);
1650
1651         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1652
1653         return common_nonsnoop(edesc, areq, ablkcipher_done);
1654 }
1655
1656 static void common_nonsnoop_hash_unmap(struct device *dev,
1657                                        struct talitos_edesc *edesc,
1658                                        struct ahash_request *areq)
1659 {
1660         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1661         struct talitos_private *priv = dev_get_drvdata(dev);
1662         bool is_sec1 = has_ftr_sec1(priv);
1663
1664         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1665
1666         unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1667
1668         /* When using hashctx-in, must unmap it. */
1669         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1670                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1671                                          DMA_TO_DEVICE);
1672
1673         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1674                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1675                                          DMA_TO_DEVICE);
1676
1677         if (edesc->dma_len)
1678                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1679                                  DMA_BIDIRECTIONAL);
1680
1681 }
1682
1683 static void ahash_done(struct device *dev,
1684                        struct talitos_desc *desc, void *context,
1685                        int err)
1686 {
1687         struct ahash_request *areq = context;
1688         struct talitos_edesc *edesc =
1689                  container_of(desc, struct talitos_edesc, desc);
1690         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1691
1692         if (!req_ctx->last && req_ctx->to_hash_later) {
1693                 /* Position any partial block for next update/final/finup */
1694                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1695                 req_ctx->nbuf = req_ctx->to_hash_later;
1696         }
1697         common_nonsnoop_hash_unmap(dev, edesc, areq);
1698
1699         kfree(edesc);
1700
1701         areq->base.complete(&areq->base, err);
1702 }
1703
1704 /*
1705  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1706  * ourself and submit a padded block
1707  */
1708 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1709                                struct talitos_edesc *edesc,
1710                                struct talitos_ptr *ptr)
1711 {
1712         static u8 padded_hash[64] = {
1713                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1714                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1715                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1716                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1717         };
1718
1719         pr_err_once("Bug in SEC1, padding ourself\n");
1720         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1721         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1722                                (char *)padded_hash, DMA_TO_DEVICE);
1723 }
1724
1725 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1726                                 struct ahash_request *areq, unsigned int length,
1727                                 void (*callback) (struct device *dev,
1728                                                   struct talitos_desc *desc,
1729                                                   void *context, int error))
1730 {
1731         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1732         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1733         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1734         struct device *dev = ctx->dev;
1735         struct talitos_desc *desc = &edesc->desc;
1736         int ret;
1737         struct talitos_private *priv = dev_get_drvdata(dev);
1738         bool is_sec1 = has_ftr_sec1(priv);
1739
1740         /* first DWORD empty */
1741         desc->ptr[0] = zero_entry;
1742
1743         /* hash context in */
1744         if (!req_ctx->first || req_ctx->swinit) {
1745                 map_single_talitos_ptr(dev, &desc->ptr[1],
1746                                        req_ctx->hw_context_size,
1747                                        (char *)req_ctx->hw_context,
1748                                        DMA_TO_DEVICE);
1749                 req_ctx->swinit = 0;
1750         } else {
1751                 desc->ptr[1] = zero_entry;
1752         }
1753         /* Indicate next op is not the first. */
1754         req_ctx->first = 0;
1755
1756         /* HMAC key */
1757         if (ctx->keylen)
1758                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1759                                        (char *)&ctx->key, DMA_TO_DEVICE);
1760         else
1761                 desc->ptr[2] = zero_entry;
1762
1763         /*
1764          * data in
1765          */
1766         map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1767                               DMA_TO_DEVICE, &desc->ptr[3]);
1768
1769         /* fifth DWORD empty */
1770         desc->ptr[4] = zero_entry;
1771
1772         /* hash/HMAC out -or- hash context out */
1773         if (req_ctx->last)
1774                 map_single_talitos_ptr(dev, &desc->ptr[5],
1775                                        crypto_ahash_digestsize(tfm),
1776                                        areq->result, DMA_FROM_DEVICE);
1777         else
1778                 map_single_talitos_ptr(dev, &desc->ptr[5],
1779                                        req_ctx->hw_context_size,
1780                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1781
1782         /* last DWORD empty */
1783         desc->ptr[6] = zero_entry;
1784
1785         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1786                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1787
1788         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1789         if (ret != -EINPROGRESS) {
1790                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1791                 kfree(edesc);
1792         }
1793         return ret;
1794 }
1795
1796 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1797                                                unsigned int nbytes)
1798 {
1799         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1800         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1801         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1802
1803         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1804                                    nbytes, 0, 0, 0, areq->base.flags, false);
1805 }
1806
1807 static int ahash_init(struct ahash_request *areq)
1808 {
1809         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1810         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1811
1812         /* Initialize the context */
1813         req_ctx->nbuf = 0;
1814         req_ctx->first = 1; /* first indicates h/w must init its context */
1815         req_ctx->swinit = 0; /* assume h/w init of context */
1816         req_ctx->hw_context_size =
1817                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1818                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1819                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1820
1821         return 0;
1822 }
1823
1824 /*
1825  * on h/w without explicit sha224 support, we initialize h/w context
1826  * manually with sha224 constants, and tell it to run sha256.
1827  */
1828 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1829 {
1830         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1831
1832         ahash_init(areq);
1833         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1834
1835         req_ctx->hw_context[0] = SHA224_H0;
1836         req_ctx->hw_context[1] = SHA224_H1;
1837         req_ctx->hw_context[2] = SHA224_H2;
1838         req_ctx->hw_context[3] = SHA224_H3;
1839         req_ctx->hw_context[4] = SHA224_H4;
1840         req_ctx->hw_context[5] = SHA224_H5;
1841         req_ctx->hw_context[6] = SHA224_H6;
1842         req_ctx->hw_context[7] = SHA224_H7;
1843
1844         /* init 64-bit count */
1845         req_ctx->hw_context[8] = 0;
1846         req_ctx->hw_context[9] = 0;
1847
1848         return 0;
1849 }
1850
1851 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1852 {
1853         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1854         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1855         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1856         struct talitos_edesc *edesc;
1857         unsigned int blocksize =
1858                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1859         unsigned int nbytes_to_hash;
1860         unsigned int to_hash_later;
1861         unsigned int nsg;
1862
1863         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1864                 /* Buffer up to one whole block */
1865                 sg_copy_to_buffer(areq->src,
1866                                   sg_nents_for_len(areq->src, nbytes),
1867                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1868                 req_ctx->nbuf += nbytes;
1869                 return 0;
1870         }
1871
1872         /* At least (blocksize + 1) bytes are available to hash */
1873         nbytes_to_hash = nbytes + req_ctx->nbuf;
1874         to_hash_later = nbytes_to_hash & (blocksize - 1);
1875
1876         if (req_ctx->last)
1877                 to_hash_later = 0;
1878         else if (to_hash_later)
1879                 /* There is a partial block. Hash the full block(s) now */
1880                 nbytes_to_hash -= to_hash_later;
1881         else {
1882                 /* Keep one block buffered */
1883                 nbytes_to_hash -= blocksize;
1884                 to_hash_later = blocksize;
1885         }
1886
1887         /* Chain in any previously buffered data */
1888         if (req_ctx->nbuf) {
1889                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1890                 sg_init_table(req_ctx->bufsl, nsg);
1891                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1892                 if (nsg > 1)
1893                         sg_chain(req_ctx->bufsl, 2, areq->src);
1894                 req_ctx->psrc = req_ctx->bufsl;
1895         } else
1896                 req_ctx->psrc = areq->src;
1897
1898         if (to_hash_later) {
1899                 int nents = sg_nents_for_len(areq->src, nbytes);
1900                 sg_pcopy_to_buffer(areq->src, nents,
1901                                       req_ctx->bufnext,
1902                                       to_hash_later,
1903                                       nbytes - to_hash_later);
1904         }
1905         req_ctx->to_hash_later = to_hash_later;
1906
1907         /* Allocate extended descriptor */
1908         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1909         if (IS_ERR(edesc))
1910                 return PTR_ERR(edesc);
1911
1912         edesc->desc.hdr = ctx->desc_hdr_template;
1913
1914         /* On last one, request SEC to pad; otherwise continue */
1915         if (req_ctx->last)
1916                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1917         else
1918                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1919
1920         /* request SEC to INIT hash. */
1921         if (req_ctx->first && !req_ctx->swinit)
1922                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1923
1924         /* When the tfm context has a keylen, it's an HMAC.
1925          * A first or last (ie. not middle) descriptor must request HMAC.
1926          */
1927         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1928                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1929
1930         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1931                                     ahash_done);
1932 }
1933
1934 static int ahash_update(struct ahash_request *areq)
1935 {
1936         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1937
1938         req_ctx->last = 0;
1939
1940         return ahash_process_req(areq, areq->nbytes);
1941 }
1942
1943 static int ahash_final(struct ahash_request *areq)
1944 {
1945         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1946
1947         req_ctx->last = 1;
1948
1949         return ahash_process_req(areq, 0);
1950 }
1951
1952 static int ahash_finup(struct ahash_request *areq)
1953 {
1954         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1955
1956         req_ctx->last = 1;
1957
1958         return ahash_process_req(areq, areq->nbytes);
1959 }
1960
1961 static int ahash_digest(struct ahash_request *areq)
1962 {
1963         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1964         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1965
1966         ahash->init(areq);
1967         req_ctx->last = 1;
1968
1969         return ahash_process_req(areq, areq->nbytes);
1970 }
1971
1972 static int ahash_export(struct ahash_request *areq, void *out)
1973 {
1974         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1975         struct talitos_export_state *export = out;
1976
1977         memcpy(export->hw_context, req_ctx->hw_context,
1978                req_ctx->hw_context_size);
1979         memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
1980         export->swinit = req_ctx->swinit;
1981         export->first = req_ctx->first;
1982         export->last = req_ctx->last;
1983         export->to_hash_later = req_ctx->to_hash_later;
1984         export->nbuf = req_ctx->nbuf;
1985
1986         return 0;
1987 }
1988
1989 static int ahash_import(struct ahash_request *areq, const void *in)
1990 {
1991         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1992         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1993         const struct talitos_export_state *export = in;
1994
1995         memset(req_ctx, 0, sizeof(*req_ctx));
1996         req_ctx->hw_context_size =
1997                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1998                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1999                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2000         memcpy(req_ctx->hw_context, export->hw_context,
2001                req_ctx->hw_context_size);
2002         memcpy(req_ctx->buf, export->buf, export->nbuf);
2003         req_ctx->swinit = export->swinit;
2004         req_ctx->first = export->first;
2005         req_ctx->last = export->last;
2006         req_ctx->to_hash_later = export->to_hash_later;
2007         req_ctx->nbuf = export->nbuf;
2008
2009         return 0;
2010 }
2011
2012 struct keyhash_result {
2013         struct completion completion;
2014         int err;
2015 };
2016
2017 static void keyhash_complete(struct crypto_async_request *req, int err)
2018 {
2019         struct keyhash_result *res = req->data;
2020
2021         if (err == -EINPROGRESS)
2022                 return;
2023
2024         res->err = err;
2025         complete(&res->completion);
2026 }
2027
2028 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2029                    u8 *hash)
2030 {
2031         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2032
2033         struct scatterlist sg[1];
2034         struct ahash_request *req;
2035         struct keyhash_result hresult;
2036         int ret;
2037
2038         init_completion(&hresult.completion);
2039
2040         req = ahash_request_alloc(tfm, GFP_KERNEL);
2041         if (!req)
2042                 return -ENOMEM;
2043
2044         /* Keep tfm keylen == 0 during hash of the long key */
2045         ctx->keylen = 0;
2046         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2047                                    keyhash_complete, &hresult);
2048
2049         sg_init_one(&sg[0], key, keylen);
2050
2051         ahash_request_set_crypt(req, sg, hash, keylen);
2052         ret = crypto_ahash_digest(req);
2053         switch (ret) {
2054         case 0:
2055                 break;
2056         case -EINPROGRESS:
2057         case -EBUSY:
2058                 ret = wait_for_completion_interruptible(
2059                         &hresult.completion);
2060                 if (!ret)
2061                         ret = hresult.err;
2062                 break;
2063         default:
2064                 break;
2065         }
2066         ahash_request_free(req);
2067
2068         return ret;
2069 }
2070
2071 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2072                         unsigned int keylen)
2073 {
2074         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2075         unsigned int blocksize =
2076                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2077         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2078         unsigned int keysize = keylen;
2079         u8 hash[SHA512_DIGEST_SIZE];
2080         int ret;
2081
2082         if (keylen <= blocksize)
2083                 memcpy(ctx->key, key, keysize);
2084         else {
2085                 /* Must get the hash of the long key */
2086                 ret = keyhash(tfm, key, keylen, hash);
2087
2088                 if (ret) {
2089                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2090                         return -EINVAL;
2091                 }
2092
2093                 keysize = digestsize;
2094                 memcpy(ctx->key, hash, digestsize);
2095         }
2096
2097         ctx->keylen = keysize;
2098
2099         return 0;
2100 }
2101
2102
2103 struct talitos_alg_template {
2104         u32 type;
2105         union {
2106                 struct crypto_alg crypto;
2107                 struct ahash_alg hash;
2108                 struct aead_alg aead;
2109         } alg;
2110         __be32 desc_hdr_template;
2111 };
2112
2113 static struct talitos_alg_template driver_algs[] = {
2114         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2115         {       .type = CRYPTO_ALG_TYPE_AEAD,
2116                 .alg.aead = {
2117                         .base = {
2118                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2119                                 .cra_driver_name = "authenc-hmac-sha1-"
2120                                                    "cbc-aes-talitos",
2121                                 .cra_blocksize = AES_BLOCK_SIZE,
2122                                 .cra_flags = CRYPTO_ALG_ASYNC,
2123                         },
2124                         .ivsize = AES_BLOCK_SIZE,
2125                         .maxauthsize = SHA1_DIGEST_SIZE,
2126                 },
2127                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2128                                      DESC_HDR_SEL0_AESU |
2129                                      DESC_HDR_MODE0_AESU_CBC |
2130                                      DESC_HDR_SEL1_MDEUA |
2131                                      DESC_HDR_MODE1_MDEU_INIT |
2132                                      DESC_HDR_MODE1_MDEU_PAD |
2133                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2134         },
2135         {       .type = CRYPTO_ALG_TYPE_AEAD,
2136                 .alg.aead = {
2137                         .base = {
2138                                 .cra_name = "authenc(hmac(sha1),"
2139                                             "cbc(des3_ede))",
2140                                 .cra_driver_name = "authenc-hmac-sha1-"
2141                                                    "cbc-3des-talitos",
2142                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2143                                 .cra_flags = CRYPTO_ALG_ASYNC,
2144                         },
2145                         .ivsize = DES3_EDE_BLOCK_SIZE,
2146                         .maxauthsize = SHA1_DIGEST_SIZE,
2147                 },
2148                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2149                                      DESC_HDR_SEL0_DEU |
2150                                      DESC_HDR_MODE0_DEU_CBC |
2151                                      DESC_HDR_MODE0_DEU_3DES |
2152                                      DESC_HDR_SEL1_MDEUA |
2153                                      DESC_HDR_MODE1_MDEU_INIT |
2154                                      DESC_HDR_MODE1_MDEU_PAD |
2155                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2156         },
2157         {       .type = CRYPTO_ALG_TYPE_AEAD,
2158                 .alg.aead = {
2159                         .base = {
2160                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2161                                 .cra_driver_name = "authenc-hmac-sha224-"
2162                                                    "cbc-aes-talitos",
2163                                 .cra_blocksize = AES_BLOCK_SIZE,
2164                                 .cra_flags = CRYPTO_ALG_ASYNC,
2165                         },
2166                         .ivsize = AES_BLOCK_SIZE,
2167                         .maxauthsize = SHA224_DIGEST_SIZE,
2168                 },
2169                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2170                                      DESC_HDR_SEL0_AESU |
2171                                      DESC_HDR_MODE0_AESU_CBC |
2172                                      DESC_HDR_SEL1_MDEUA |
2173                                      DESC_HDR_MODE1_MDEU_INIT |
2174                                      DESC_HDR_MODE1_MDEU_PAD |
2175                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2176         },
2177         {       .type = CRYPTO_ALG_TYPE_AEAD,
2178                 .alg.aead = {
2179                         .base = {
2180                                 .cra_name = "authenc(hmac(sha224),"
2181                                             "cbc(des3_ede))",
2182                                 .cra_driver_name = "authenc-hmac-sha224-"
2183                                                    "cbc-3des-talitos",
2184                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2185                                 .cra_flags = CRYPTO_ALG_ASYNC,
2186                         },
2187                         .ivsize = DES3_EDE_BLOCK_SIZE,
2188                         .maxauthsize = SHA224_DIGEST_SIZE,
2189                 },
2190                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2191                                      DESC_HDR_SEL0_DEU |
2192                                      DESC_HDR_MODE0_DEU_CBC |
2193                                      DESC_HDR_MODE0_DEU_3DES |
2194                                      DESC_HDR_SEL1_MDEUA |
2195                                      DESC_HDR_MODE1_MDEU_INIT |
2196                                      DESC_HDR_MODE1_MDEU_PAD |
2197                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2198         },
2199         {       .type = CRYPTO_ALG_TYPE_AEAD,
2200                 .alg.aead = {
2201                         .base = {
2202                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2203                                 .cra_driver_name = "authenc-hmac-sha256-"
2204                                                    "cbc-aes-talitos",
2205                                 .cra_blocksize = AES_BLOCK_SIZE,
2206                                 .cra_flags = CRYPTO_ALG_ASYNC,
2207                         },
2208                         .ivsize = AES_BLOCK_SIZE,
2209                         .maxauthsize = SHA256_DIGEST_SIZE,
2210                 },
2211                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2212                                      DESC_HDR_SEL0_AESU |
2213                                      DESC_HDR_MODE0_AESU_CBC |
2214                                      DESC_HDR_SEL1_MDEUA |
2215                                      DESC_HDR_MODE1_MDEU_INIT |
2216                                      DESC_HDR_MODE1_MDEU_PAD |
2217                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2218         },
2219         {       .type = CRYPTO_ALG_TYPE_AEAD,
2220                 .alg.aead = {
2221                         .base = {
2222                                 .cra_name = "authenc(hmac(sha256),"
2223                                             "cbc(des3_ede))",
2224                                 .cra_driver_name = "authenc-hmac-sha256-"
2225                                                    "cbc-3des-talitos",
2226                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2227                                 .cra_flags = CRYPTO_ALG_ASYNC,
2228                         },
2229                         .ivsize = DES3_EDE_BLOCK_SIZE,
2230                         .maxauthsize = SHA256_DIGEST_SIZE,
2231                 },
2232                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2233                                      DESC_HDR_SEL0_DEU |
2234                                      DESC_HDR_MODE0_DEU_CBC |
2235                                      DESC_HDR_MODE0_DEU_3DES |
2236                                      DESC_HDR_SEL1_MDEUA |
2237                                      DESC_HDR_MODE1_MDEU_INIT |
2238                                      DESC_HDR_MODE1_MDEU_PAD |
2239                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2240         },
2241         {       .type = CRYPTO_ALG_TYPE_AEAD,
2242                 .alg.aead = {
2243                         .base = {
2244                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2245                                 .cra_driver_name = "authenc-hmac-sha384-"
2246                                                    "cbc-aes-talitos",
2247                                 .cra_blocksize = AES_BLOCK_SIZE,
2248                                 .cra_flags = CRYPTO_ALG_ASYNC,
2249                         },
2250                         .ivsize = AES_BLOCK_SIZE,
2251                         .maxauthsize = SHA384_DIGEST_SIZE,
2252                 },
2253                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2254                                      DESC_HDR_SEL0_AESU |
2255                                      DESC_HDR_MODE0_AESU_CBC |
2256                                      DESC_HDR_SEL1_MDEUB |
2257                                      DESC_HDR_MODE1_MDEU_INIT |
2258                                      DESC_HDR_MODE1_MDEU_PAD |
2259                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2260         },
2261         {       .type = CRYPTO_ALG_TYPE_AEAD,
2262                 .alg.aead = {
2263                         .base = {
2264                                 .cra_name = "authenc(hmac(sha384),"
2265                                             "cbc(des3_ede))",
2266                                 .cra_driver_name = "authenc-hmac-sha384-"
2267                                                    "cbc-3des-talitos",
2268                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2269                                 .cra_flags = CRYPTO_ALG_ASYNC,
2270                         },
2271                         .ivsize = DES3_EDE_BLOCK_SIZE,
2272                         .maxauthsize = SHA384_DIGEST_SIZE,
2273                 },
2274                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2275                                      DESC_HDR_SEL0_DEU |
2276                                      DESC_HDR_MODE0_DEU_CBC |
2277                                      DESC_HDR_MODE0_DEU_3DES |
2278                                      DESC_HDR_SEL1_MDEUB |
2279                                      DESC_HDR_MODE1_MDEU_INIT |
2280                                      DESC_HDR_MODE1_MDEU_PAD |
2281                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2282         },
2283         {       .type = CRYPTO_ALG_TYPE_AEAD,
2284                 .alg.aead = {
2285                         .base = {
2286                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2287                                 .cra_driver_name = "authenc-hmac-sha512-"
2288                                                    "cbc-aes-talitos",
2289                                 .cra_blocksize = AES_BLOCK_SIZE,
2290                                 .cra_flags = CRYPTO_ALG_ASYNC,
2291                         },
2292                         .ivsize = AES_BLOCK_SIZE,
2293                         .maxauthsize = SHA512_DIGEST_SIZE,
2294                 },
2295                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2296                                      DESC_HDR_SEL0_AESU |
2297                                      DESC_HDR_MODE0_AESU_CBC |
2298                                      DESC_HDR_SEL1_MDEUB |
2299                                      DESC_HDR_MODE1_MDEU_INIT |
2300                                      DESC_HDR_MODE1_MDEU_PAD |
2301                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2302         },
2303         {       .type = CRYPTO_ALG_TYPE_AEAD,
2304                 .alg.aead = {
2305                         .base = {
2306                                 .cra_name = "authenc(hmac(sha512),"
2307                                             "cbc(des3_ede))",
2308                                 .cra_driver_name = "authenc-hmac-sha512-"
2309                                                    "cbc-3des-talitos",
2310                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2311                                 .cra_flags = CRYPTO_ALG_ASYNC,
2312                         },
2313                         .ivsize = DES3_EDE_BLOCK_SIZE,
2314                         .maxauthsize = SHA512_DIGEST_SIZE,
2315                 },
2316                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2317                                      DESC_HDR_SEL0_DEU |
2318                                      DESC_HDR_MODE0_DEU_CBC |
2319                                      DESC_HDR_MODE0_DEU_3DES |
2320                                      DESC_HDR_SEL1_MDEUB |
2321                                      DESC_HDR_MODE1_MDEU_INIT |
2322                                      DESC_HDR_MODE1_MDEU_PAD |
2323                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2324         },
2325         {       .type = CRYPTO_ALG_TYPE_AEAD,
2326                 .alg.aead = {
2327                         .base = {
2328                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2329                                 .cra_driver_name = "authenc-hmac-md5-"
2330                                                    "cbc-aes-talitos",
2331                                 .cra_blocksize = AES_BLOCK_SIZE,
2332                                 .cra_flags = CRYPTO_ALG_ASYNC,
2333                         },
2334                         .ivsize = AES_BLOCK_SIZE,
2335                         .maxauthsize = MD5_DIGEST_SIZE,
2336                 },
2337                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2338                                      DESC_HDR_SEL0_AESU |
2339                                      DESC_HDR_MODE0_AESU_CBC |
2340                                      DESC_HDR_SEL1_MDEUA |
2341                                      DESC_HDR_MODE1_MDEU_INIT |
2342                                      DESC_HDR_MODE1_MDEU_PAD |
2343                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2344         },
2345         {       .type = CRYPTO_ALG_TYPE_AEAD,
2346                 .alg.aead = {
2347                         .base = {
2348                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2349                                 .cra_driver_name = "authenc-hmac-md5-"
2350                                                    "cbc-3des-talitos",
2351                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2352                                 .cra_flags = CRYPTO_ALG_ASYNC,
2353                         },
2354                         .ivsize = DES3_EDE_BLOCK_SIZE,
2355                         .maxauthsize = MD5_DIGEST_SIZE,
2356                 },
2357                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2358                                      DESC_HDR_SEL0_DEU |
2359                                      DESC_HDR_MODE0_DEU_CBC |
2360                                      DESC_HDR_MODE0_DEU_3DES |
2361                                      DESC_HDR_SEL1_MDEUA |
2362                                      DESC_HDR_MODE1_MDEU_INIT |
2363                                      DESC_HDR_MODE1_MDEU_PAD |
2364                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2365         },
2366         /* ABLKCIPHER algorithms. */
2367         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2368                 .alg.crypto = {
2369                         .cra_name = "cbc(aes)",
2370                         .cra_driver_name = "cbc-aes-talitos",
2371                         .cra_blocksize = AES_BLOCK_SIZE,
2372                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2373                                      CRYPTO_ALG_ASYNC,
2374                         .cra_ablkcipher = {
2375                                 .min_keysize = AES_MIN_KEY_SIZE,
2376                                 .max_keysize = AES_MAX_KEY_SIZE,
2377                                 .ivsize = AES_BLOCK_SIZE,
2378                         }
2379                 },
2380                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2381                                      DESC_HDR_SEL0_AESU |
2382                                      DESC_HDR_MODE0_AESU_CBC,
2383         },
2384         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2385                 .alg.crypto = {
2386                         .cra_name = "cbc(des3_ede)",
2387                         .cra_driver_name = "cbc-3des-talitos",
2388                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2389                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2390                                      CRYPTO_ALG_ASYNC,
2391                         .cra_ablkcipher = {
2392                                 .min_keysize = DES3_EDE_KEY_SIZE,
2393                                 .max_keysize = DES3_EDE_KEY_SIZE,
2394                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2395                         }
2396                 },
2397                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2398                                      DESC_HDR_SEL0_DEU |
2399                                      DESC_HDR_MODE0_DEU_CBC |
2400                                      DESC_HDR_MODE0_DEU_3DES,
2401         },
2402         /* AHASH algorithms. */
2403         {       .type = CRYPTO_ALG_TYPE_AHASH,
2404                 .alg.hash = {
2405                         .halg.digestsize = MD5_DIGEST_SIZE,
2406                         .halg.statesize = sizeof(struct talitos_export_state),
2407                         .halg.base = {
2408                                 .cra_name = "md5",
2409                                 .cra_driver_name = "md5-talitos",
2410                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2411                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2412                                              CRYPTO_ALG_ASYNC,
2413                         }
2414                 },
2415                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2416                                      DESC_HDR_SEL0_MDEUA |
2417                                      DESC_HDR_MODE0_MDEU_MD5,
2418         },
2419         {       .type = CRYPTO_ALG_TYPE_AHASH,
2420                 .alg.hash = {
2421                         .halg.digestsize = SHA1_DIGEST_SIZE,
2422                         .halg.statesize = sizeof(struct talitos_export_state),
2423                         .halg.base = {
2424                                 .cra_name = "sha1",
2425                                 .cra_driver_name = "sha1-talitos",
2426                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2427                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2428                                              CRYPTO_ALG_ASYNC,
2429                         }
2430                 },
2431                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2432                                      DESC_HDR_SEL0_MDEUA |
2433                                      DESC_HDR_MODE0_MDEU_SHA1,
2434         },
2435         {       .type = CRYPTO_ALG_TYPE_AHASH,
2436                 .alg.hash = {
2437                         .halg.digestsize = SHA224_DIGEST_SIZE,
2438                         .halg.statesize = sizeof(struct talitos_export_state),
2439                         .halg.base = {
2440                                 .cra_name = "sha224",
2441                                 .cra_driver_name = "sha224-talitos",
2442                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2443                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2444                                              CRYPTO_ALG_ASYNC,
2445                         }
2446                 },
2447                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2448                                      DESC_HDR_SEL0_MDEUA |
2449                                      DESC_HDR_MODE0_MDEU_SHA224,
2450         },
2451         {       .type = CRYPTO_ALG_TYPE_AHASH,
2452                 .alg.hash = {
2453                         .halg.digestsize = SHA256_DIGEST_SIZE,
2454                         .halg.statesize = sizeof(struct talitos_export_state),
2455                         .halg.base = {
2456                                 .cra_name = "sha256",
2457                                 .cra_driver_name = "sha256-talitos",
2458                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2459                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2460                                              CRYPTO_ALG_ASYNC,
2461                         }
2462                 },
2463                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2464                                      DESC_HDR_SEL0_MDEUA |
2465                                      DESC_HDR_MODE0_MDEU_SHA256,
2466         },
2467         {       .type = CRYPTO_ALG_TYPE_AHASH,
2468                 .alg.hash = {
2469                         .halg.digestsize = SHA384_DIGEST_SIZE,
2470                         .halg.statesize = sizeof(struct talitos_export_state),
2471                         .halg.base = {
2472                                 .cra_name = "sha384",
2473                                 .cra_driver_name = "sha384-talitos",
2474                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2475                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2476                                              CRYPTO_ALG_ASYNC,
2477                         }
2478                 },
2479                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2480                                      DESC_HDR_SEL0_MDEUB |
2481                                      DESC_HDR_MODE0_MDEUB_SHA384,
2482         },
2483         {       .type = CRYPTO_ALG_TYPE_AHASH,
2484                 .alg.hash = {
2485                         .halg.digestsize = SHA512_DIGEST_SIZE,
2486                         .halg.statesize = sizeof(struct talitos_export_state),
2487                         .halg.base = {
2488                                 .cra_name = "sha512",
2489                                 .cra_driver_name = "sha512-talitos",
2490                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2491                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2492                                              CRYPTO_ALG_ASYNC,
2493                         }
2494                 },
2495                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2496                                      DESC_HDR_SEL0_MDEUB |
2497                                      DESC_HDR_MODE0_MDEUB_SHA512,
2498         },
2499         {       .type = CRYPTO_ALG_TYPE_AHASH,
2500                 .alg.hash = {
2501                         .halg.digestsize = MD5_DIGEST_SIZE,
2502                         .halg.statesize = sizeof(struct talitos_export_state),
2503                         .halg.base = {
2504                                 .cra_name = "hmac(md5)",
2505                                 .cra_driver_name = "hmac-md5-talitos",
2506                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2507                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2508                                              CRYPTO_ALG_ASYNC,
2509                         }
2510                 },
2511                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2512                                      DESC_HDR_SEL0_MDEUA |
2513                                      DESC_HDR_MODE0_MDEU_MD5,
2514         },
2515         {       .type = CRYPTO_ALG_TYPE_AHASH,
2516                 .alg.hash = {
2517                         .halg.digestsize = SHA1_DIGEST_SIZE,
2518                         .halg.statesize = sizeof(struct talitos_export_state),
2519                         .halg.base = {
2520                                 .cra_name = "hmac(sha1)",
2521                                 .cra_driver_name = "hmac-sha1-talitos",
2522                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2523                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2524                                              CRYPTO_ALG_ASYNC,
2525                         }
2526                 },
2527                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2528                                      DESC_HDR_SEL0_MDEUA |
2529                                      DESC_HDR_MODE0_MDEU_SHA1,
2530         },
2531         {       .type = CRYPTO_ALG_TYPE_AHASH,
2532                 .alg.hash = {
2533                         .halg.digestsize = SHA224_DIGEST_SIZE,
2534                         .halg.statesize = sizeof(struct talitos_export_state),
2535                         .halg.base = {
2536                                 .cra_name = "hmac(sha224)",
2537                                 .cra_driver_name = "hmac-sha224-talitos",
2538                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2539                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2540                                              CRYPTO_ALG_ASYNC,
2541                         }
2542                 },
2543                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2544                                      DESC_HDR_SEL0_MDEUA |
2545                                      DESC_HDR_MODE0_MDEU_SHA224,
2546         },
2547         {       .type = CRYPTO_ALG_TYPE_AHASH,
2548                 .alg.hash = {
2549                         .halg.digestsize = SHA256_DIGEST_SIZE,
2550                         .halg.statesize = sizeof(struct talitos_export_state),
2551                         .halg.base = {
2552                                 .cra_name = "hmac(sha256)",
2553                                 .cra_driver_name = "hmac-sha256-talitos",
2554                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2555                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2556                                              CRYPTO_ALG_ASYNC,
2557                         }
2558                 },
2559                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2560                                      DESC_HDR_SEL0_MDEUA |
2561                                      DESC_HDR_MODE0_MDEU_SHA256,
2562         },
2563         {       .type = CRYPTO_ALG_TYPE_AHASH,
2564                 .alg.hash = {
2565                         .halg.digestsize = SHA384_DIGEST_SIZE,
2566                         .halg.statesize = sizeof(struct talitos_export_state),
2567                         .halg.base = {
2568                                 .cra_name = "hmac(sha384)",
2569                                 .cra_driver_name = "hmac-sha384-talitos",
2570                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2571                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2572                                              CRYPTO_ALG_ASYNC,
2573                         }
2574                 },
2575                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2576                                      DESC_HDR_SEL0_MDEUB |
2577                                      DESC_HDR_MODE0_MDEUB_SHA384,
2578         },
2579         {       .type = CRYPTO_ALG_TYPE_AHASH,
2580                 .alg.hash = {
2581                         .halg.digestsize = SHA512_DIGEST_SIZE,
2582                         .halg.statesize = sizeof(struct talitos_export_state),
2583                         .halg.base = {
2584                                 .cra_name = "hmac(sha512)",
2585                                 .cra_driver_name = "hmac-sha512-talitos",
2586                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2587                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2588                                              CRYPTO_ALG_ASYNC,
2589                         }
2590                 },
2591                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2592                                      DESC_HDR_SEL0_MDEUB |
2593                                      DESC_HDR_MODE0_MDEUB_SHA512,
2594         }
2595 };
2596
2597 struct talitos_crypto_alg {
2598         struct list_head entry;
2599         struct device *dev;
2600         struct talitos_alg_template algt;
2601 };
2602
2603 static int talitos_init_common(struct talitos_ctx *ctx,
2604                                struct talitos_crypto_alg *talitos_alg)
2605 {
2606         struct talitos_private *priv;
2607
2608         /* update context with ptr to dev */
2609         ctx->dev = talitos_alg->dev;
2610
2611         /* assign SEC channel to tfm in round-robin fashion */
2612         priv = dev_get_drvdata(ctx->dev);
2613         ctx->ch = atomic_inc_return(&priv->last_chan) &
2614                   (priv->num_channels - 1);
2615
2616         /* copy descriptor header template value */
2617         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2618
2619         /* select done notification */
2620         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2621
2622         return 0;
2623 }
2624
2625 static int talitos_cra_init(struct crypto_tfm *tfm)
2626 {
2627         struct crypto_alg *alg = tfm->__crt_alg;
2628         struct talitos_crypto_alg *talitos_alg;
2629         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2630
2631         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2632                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2633                                            struct talitos_crypto_alg,
2634                                            algt.alg.hash);
2635         else
2636                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2637                                            algt.alg.crypto);
2638
2639         return talitos_init_common(ctx, talitos_alg);
2640 }
2641
2642 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2643 {
2644         struct aead_alg *alg = crypto_aead_alg(tfm);
2645         struct talitos_crypto_alg *talitos_alg;
2646         struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2647
2648         talitos_alg = container_of(alg, struct talitos_crypto_alg,
2649                                    algt.alg.aead);
2650
2651         return talitos_init_common(ctx, talitos_alg);
2652 }
2653
2654 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2655 {
2656         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2657
2658         talitos_cra_init(tfm);
2659
2660         ctx->keylen = 0;
2661         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2662                                  sizeof(struct talitos_ahash_req_ctx));
2663
2664         return 0;
2665 }
2666
2667 /*
2668  * given the alg's descriptor header template, determine whether descriptor
2669  * type and primary/secondary execution units required match the hw
2670  * capabilities description provided in the device tree node.
2671  */
2672 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2673 {
2674         struct talitos_private *priv = dev_get_drvdata(dev);
2675         int ret;
2676
2677         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2678               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2679
2680         if (SECONDARY_EU(desc_hdr_template))
2681                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2682                               & priv->exec_units);
2683
2684         return ret;
2685 }
2686
2687 static int talitos_remove(struct platform_device *ofdev)
2688 {
2689         struct device *dev = &ofdev->dev;
2690         struct talitos_private *priv = dev_get_drvdata(dev);
2691         struct talitos_crypto_alg *t_alg, *n;
2692         int i;
2693
2694         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2695                 switch (t_alg->algt.type) {
2696                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2697                         break;
2698                 case CRYPTO_ALG_TYPE_AEAD:
2699                         crypto_unregister_aead(&t_alg->algt.alg.aead);
2700                 case CRYPTO_ALG_TYPE_AHASH:
2701                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2702                         break;
2703                 }
2704                 list_del(&t_alg->entry);
2705                 kfree(t_alg);
2706         }
2707
2708         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2709                 talitos_unregister_rng(dev);
2710
2711         for (i = 0; priv->chan && i < priv->num_channels; i++)
2712                 kfree(priv->chan[i].fifo);
2713
2714         kfree(priv->chan);
2715
2716         for (i = 0; i < 2; i++)
2717                 if (priv->irq[i]) {
2718                         free_irq(priv->irq[i], dev);
2719                         irq_dispose_mapping(priv->irq[i]);
2720                 }
2721
2722         tasklet_kill(&priv->done_task[0]);
2723         if (priv->irq[1])
2724                 tasklet_kill(&priv->done_task[1]);
2725
2726         iounmap(priv->reg);
2727
2728         kfree(priv);
2729
2730         return 0;
2731 }
2732
2733 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2734                                                     struct talitos_alg_template
2735                                                            *template)
2736 {
2737         struct talitos_private *priv = dev_get_drvdata(dev);
2738         struct talitos_crypto_alg *t_alg;
2739         struct crypto_alg *alg;
2740
2741         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2742         if (!t_alg)
2743                 return ERR_PTR(-ENOMEM);
2744
2745         t_alg->algt = *template;
2746
2747         switch (t_alg->algt.type) {
2748         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2749                 alg = &t_alg->algt.alg.crypto;
2750                 alg->cra_init = talitos_cra_init;
2751                 alg->cra_type = &crypto_ablkcipher_type;
2752                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2753                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2754                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2755                 alg->cra_ablkcipher.geniv = "eseqiv";
2756                 break;
2757         case CRYPTO_ALG_TYPE_AEAD:
2758                 alg = &t_alg->algt.alg.aead.base;
2759                 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
2760                 t_alg->algt.alg.aead.setkey = aead_setkey;
2761                 t_alg->algt.alg.aead.encrypt = aead_encrypt;
2762                 t_alg->algt.alg.aead.decrypt = aead_decrypt;
2763                 break;
2764         case CRYPTO_ALG_TYPE_AHASH:
2765                 alg = &t_alg->algt.alg.hash.halg.base;
2766                 alg->cra_init = talitos_cra_init_ahash;
2767                 alg->cra_type = &crypto_ahash_type;
2768                 t_alg->algt.alg.hash.init = ahash_init;
2769                 t_alg->algt.alg.hash.update = ahash_update;
2770                 t_alg->algt.alg.hash.final = ahash_final;
2771                 t_alg->algt.alg.hash.finup = ahash_finup;
2772                 t_alg->algt.alg.hash.digest = ahash_digest;
2773                 if (!strncmp(alg->cra_name, "hmac", 4))
2774                         t_alg->algt.alg.hash.setkey = ahash_setkey;
2775                 t_alg->algt.alg.hash.import = ahash_import;
2776                 t_alg->algt.alg.hash.export = ahash_export;
2777
2778                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2779                     !strncmp(alg->cra_name, "hmac", 4)) {
2780                         kfree(t_alg);
2781                         return ERR_PTR(-ENOTSUPP);
2782                 }
2783                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2784                     (!strcmp(alg->cra_name, "sha224") ||
2785                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2786                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2787                         t_alg->algt.desc_hdr_template =
2788                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2789                                         DESC_HDR_SEL0_MDEUA |
2790                                         DESC_HDR_MODE0_MDEU_SHA256;
2791                 }
2792                 break;
2793         default:
2794                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2795                 kfree(t_alg);
2796                 return ERR_PTR(-EINVAL);
2797         }
2798
2799         alg->cra_module = THIS_MODULE;
2800         alg->cra_priority = TALITOS_CRA_PRIORITY;
2801         alg->cra_alignmask = 0;
2802         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2803         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2804
2805         t_alg->dev = dev;
2806
2807         return t_alg;
2808 }
2809
2810 static int talitos_probe_irq(struct platform_device *ofdev)
2811 {
2812         struct device *dev = &ofdev->dev;
2813         struct device_node *np = ofdev->dev.of_node;
2814         struct talitos_private *priv = dev_get_drvdata(dev);
2815         int err;
2816         bool is_sec1 = has_ftr_sec1(priv);
2817
2818         priv->irq[0] = irq_of_parse_and_map(np, 0);
2819         if (!priv->irq[0]) {
2820                 dev_err(dev, "failed to map irq\n");
2821                 return -EINVAL;
2822         }
2823         if (is_sec1) {
2824                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2825                                   dev_driver_string(dev), dev);
2826                 goto primary_out;
2827         }
2828
2829         priv->irq[1] = irq_of_parse_and_map(np, 1);
2830
2831         /* get the primary irq line */
2832         if (!priv->irq[1]) {
2833                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
2834                                   dev_driver_string(dev), dev);
2835                 goto primary_out;
2836         }
2837
2838         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
2839                           dev_driver_string(dev), dev);
2840         if (err)
2841                 goto primary_out;
2842
2843         /* get the secondary irq line */
2844         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
2845                           dev_driver_string(dev), dev);
2846         if (err) {
2847                 dev_err(dev, "failed to request secondary irq\n");
2848                 irq_dispose_mapping(priv->irq[1]);
2849                 priv->irq[1] = 0;
2850         }
2851
2852         return err;
2853
2854 primary_out:
2855         if (err) {
2856                 dev_err(dev, "failed to request primary irq\n");
2857                 irq_dispose_mapping(priv->irq[0]);
2858                 priv->irq[0] = 0;
2859         }
2860
2861         return err;
2862 }
2863
2864 static int talitos_probe(struct platform_device *ofdev)
2865 {
2866         struct device *dev = &ofdev->dev;
2867         struct device_node *np = ofdev->dev.of_node;
2868         struct talitos_private *priv;
2869         const unsigned int *prop;
2870         int i, err;
2871         int stride;
2872
2873         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2874         if (!priv)
2875                 return -ENOMEM;
2876
2877         INIT_LIST_HEAD(&priv->alg_list);
2878
2879         dev_set_drvdata(dev, priv);
2880
2881         priv->ofdev = ofdev;
2882
2883         spin_lock_init(&priv->reg_lock);
2884
2885         priv->reg = of_iomap(np, 0);
2886         if (!priv->reg) {
2887                 dev_err(dev, "failed to of_iomap\n");
2888                 err = -ENOMEM;
2889                 goto err_out;
2890         }
2891
2892         /* get SEC version capabilities from device tree */
2893         prop = of_get_property(np, "fsl,num-channels", NULL);
2894         if (prop)
2895                 priv->num_channels = *prop;
2896
2897         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2898         if (prop)
2899                 priv->chfifo_len = *prop;
2900
2901         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2902         if (prop)
2903                 priv->exec_units = *prop;
2904
2905         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2906         if (prop)
2907                 priv->desc_types = *prop;
2908
2909         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2910             !priv->exec_units || !priv->desc_types) {
2911                 dev_err(dev, "invalid property data in device tree node\n");
2912                 err = -EINVAL;
2913                 goto err_out;
2914         }
2915
2916         if (of_device_is_compatible(np, "fsl,sec3.0"))
2917                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2918
2919         if (of_device_is_compatible(np, "fsl,sec2.1"))
2920                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2921                                   TALITOS_FTR_SHA224_HWINIT |
2922                                   TALITOS_FTR_HMAC_OK;
2923
2924         if (of_device_is_compatible(np, "fsl,sec1.0"))
2925                 priv->features |= TALITOS_FTR_SEC1;
2926
2927         if (of_device_is_compatible(np, "fsl,sec1.2")) {
2928                 priv->reg_deu = priv->reg + TALITOS12_DEU;
2929                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
2930                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
2931                 stride = TALITOS1_CH_STRIDE;
2932         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
2933                 priv->reg_deu = priv->reg + TALITOS10_DEU;
2934                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
2935                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
2936                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
2937                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
2938                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
2939                 stride = TALITOS1_CH_STRIDE;
2940         } else {
2941                 priv->reg_deu = priv->reg + TALITOS2_DEU;
2942                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
2943                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
2944                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
2945                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
2946                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
2947                 priv->reg_keu = priv->reg + TALITOS2_KEU;
2948                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
2949                 stride = TALITOS2_CH_STRIDE;
2950         }
2951
2952         err = talitos_probe_irq(ofdev);
2953         if (err)
2954                 goto err_out;
2955
2956         if (of_device_is_compatible(np, "fsl,sec1.0")) {
2957                 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
2958                              (unsigned long)dev);
2959         } else {
2960                 if (!priv->irq[1]) {
2961                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
2962                                      (unsigned long)dev);
2963                 } else {
2964                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
2965                                      (unsigned long)dev);
2966                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
2967                                      (unsigned long)dev);
2968                 }
2969         }
2970
2971         priv->chan = kzalloc(sizeof(struct talitos_channel) *
2972                              priv->num_channels, GFP_KERNEL);
2973         if (!priv->chan) {
2974                 dev_err(dev, "failed to allocate channel management space\n");
2975                 err = -ENOMEM;
2976                 goto err_out;
2977         }
2978
2979         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2980
2981         for (i = 0; i < priv->num_channels; i++) {
2982                 priv->chan[i].reg = priv->reg + stride * (i + 1);
2983                 if (!priv->irq[1] || !(i & 1))
2984                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2985
2986                 spin_lock_init(&priv->chan[i].head_lock);
2987                 spin_lock_init(&priv->chan[i].tail_lock);
2988
2989                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2990                                              priv->fifo_len, GFP_KERNEL);
2991                 if (!priv->chan[i].fifo) {
2992                         dev_err(dev, "failed to allocate request fifo %d\n", i);
2993                         err = -ENOMEM;
2994                         goto err_out;
2995                 }
2996
2997                 atomic_set(&priv->chan[i].submit_count,
2998                            -(priv->chfifo_len - 1));
2999         }
3000
3001         dma_set_mask(dev, DMA_BIT_MASK(36));
3002
3003         /* reset and initialize the h/w */
3004         err = init_device(dev);
3005         if (err) {
3006                 dev_err(dev, "failed to initialize device\n");
3007                 goto err_out;
3008         }
3009
3010         /* register the RNG, if available */
3011         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3012                 err = talitos_register_rng(dev);
3013                 if (err) {
3014                         dev_err(dev, "failed to register hwrng: %d\n", err);
3015                         goto err_out;
3016                 } else
3017                         dev_info(dev, "hwrng\n");
3018         }
3019
3020         /* register crypto algorithms the device supports */
3021         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3022                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3023                         struct talitos_crypto_alg *t_alg;
3024                         struct crypto_alg *alg = NULL;
3025
3026                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3027                         if (IS_ERR(t_alg)) {
3028                                 err = PTR_ERR(t_alg);
3029                                 if (err == -ENOTSUPP)
3030                                         continue;
3031                                 goto err_out;
3032                         }
3033
3034                         switch (t_alg->algt.type) {
3035                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3036                                 err = crypto_register_alg(
3037                                                 &t_alg->algt.alg.crypto);
3038                                 alg = &t_alg->algt.alg.crypto;
3039                                 break;
3040
3041                         case CRYPTO_ALG_TYPE_AEAD:
3042                                 err = crypto_register_aead(
3043                                         &t_alg->algt.alg.aead);
3044                                 alg = &t_alg->algt.alg.aead.base;
3045                                 break;
3046
3047                         case CRYPTO_ALG_TYPE_AHASH:
3048                                 err = crypto_register_ahash(
3049                                                 &t_alg->algt.alg.hash);
3050                                 alg = &t_alg->algt.alg.hash.halg.base;
3051                                 break;
3052                         }
3053                         if (err) {
3054                                 dev_err(dev, "%s alg registration failed\n",
3055                                         alg->cra_driver_name);
3056                                 kfree(t_alg);
3057                         } else
3058                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3059                 }
3060         }
3061         if (!list_empty(&priv->alg_list))
3062                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3063                          (char *)of_get_property(np, "compatible", NULL));
3064
3065         return 0;
3066
3067 err_out:
3068         talitos_remove(ofdev);
3069
3070         return err;
3071 }
3072
3073 static const struct of_device_id talitos_match[] = {
3074 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3075         {
3076                 .compatible = "fsl,sec1.0",
3077         },
3078 #endif
3079 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3080         {
3081                 .compatible = "fsl,sec2.0",
3082         },
3083 #endif
3084         {},
3085 };
3086 MODULE_DEVICE_TABLE(of, talitos_match);
3087
3088 static struct platform_driver talitos_driver = {
3089         .driver = {
3090                 .name = "talitos",
3091                 .of_match_table = talitos_match,
3092         },
3093         .probe = talitos_probe,
3094         .remove = talitos_remove,
3095 };
3096
3097 module_platform_driver(talitos_driver);
3098
3099 MODULE_LICENSE("GPL");
3100 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3101 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");