drivers/crypto/chelsio/chcr_ktls.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
   3
   4 #ifdef CONFIG_CHELSIO_TLS_DEVICE
   5 #include <linux/highmem.h>
   6 #include "chcr_ktls.h"
   7 #include "clip_tbl.h"
   8
   9 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
  10 /*
  11  * chcr_ktls_save_keys: calculate and save crypto keys.
  12  * @tx_info - driver specific tls info.
  13  * @crypto_info - tls crypto information.
  14  * @direction - TX/RX direction.
  15  * return - SUCCESS/FAILURE.
  16  */
  17 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
  18                                struct tls_crypto_info *crypto_info,
  19                                enum tls_offload_ctx_dir direction)
  20 {
  21         int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
  22         unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
  23         struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
  24         struct ktls_key_ctx *kctx = &tx_info->key_ctx;
  25         struct crypto_cipher *cipher;
  26         unsigned char *key, *salt;
  27
  28         switch (crypto_info->cipher_type) {
  29         case TLS_CIPHER_AES_GCM_128:
  30                 info_128_gcm =
  31                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
  32                 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
  33                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
  34                 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
  35                 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
  36                 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
  37                 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
  38
  39                 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
  40                 key = info_128_gcm->key;
  41                 salt = info_128_gcm->salt;
  42                 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
  43
  44                 /* The SCMD fields used when encrypting a full TLS
  45                  * record. Its a one time calculation till the
  46                  * connection exists.
  47                  */
  48                 tx_info->scmd0_seqno_numivs =
  49                         SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
  50                         SCMD_CIPH_AUTH_SEQ_CTRL_F |
  51                         SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
  52                         SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
  53                         SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
  54                         SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
  55                         SCMD_NUM_IVS_V(1);
  56
  57                 /* keys will be sent inline. */
  58                 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
  59
  60                 /* The SCMD fields used when encrypting a partial TLS
  61                  * record (no trailer and possibly a truncated payload).
  62                  */
  63                 tx_info->scmd0_short_seqno_numivs =
  64                         SCMD_CIPH_AUTH_SEQ_CTRL_F |
  65                         SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
  66                         SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
  67                         SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
  68
  69                 tx_info->scmd0_short_ivgen_hdrlen =
  70                         tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
  71
  72                 break;
  73
  74         default:
  75                 pr_err("GCM: cipher type 0x%x not supported\n",
  76                        crypto_info->cipher_type);
  77                 ret = -EINVAL;
  78                 goto out;
  79         }
  80
  81         key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
  82                        roundup(keylen, 16) + ghash_size;
  83         /* Calculate the H = CIPH(K, 0 repeated 16 times).
  84          * It will go in key context
  85          */
  86         cipher = crypto_alloc_cipher("aes", 0, 0);
  87         if (IS_ERR(cipher)) {
  88                 ret = -ENOMEM;
  89                 goto out;
  90         }
  91
  92         ret = crypto_cipher_setkey(cipher, key, keylen);
  93         if (ret)
  94                 goto out1;
  95
  96         memset(ghash_h, 0, ghash_size);
  97         crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
  98
  99         /* fill the Key context */
 100         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 101                 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
 102                                                  mac_key_size,
 103                                                  key_ctx_size >> 4);
 104         } else {
 105                 ret = -EINVAL;
 106                 goto out1;
 107         }
 108
 109         memcpy(kctx->salt, salt, tx_info->salt_size);
 110         memcpy(kctx->key, key, keylen);
 111         memcpy(kctx->key + keylen, ghash_h, ghash_size);
 112         tx_info->key_ctx_len = key_ctx_size;
 113
 114 out1:
 115         crypto_free_cipher(cipher);
 116 out:
 117         return ret;
 118 }
 119
 120 static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
 121                                              int new_state)
 122 {
 123         unsigned long flags;
 124
 125         /* This function can be called from both rx (interrupt context) and tx
 126          * queue contexts.
 127          */
 128         spin_lock_irqsave(&tx_info->lock, flags);
 129         switch (tx_info->connection_state) {
 130         case KTLS_CONN_CLOSED:
 131                 tx_info->connection_state = new_state;
 132                 break;
 133
 134         case KTLS_CONN_ACT_OPEN_REQ:
 135                 /* only go forward if state is greater than current state. */
 136                 if (new_state <= tx_info->connection_state)
 137                         break;
 138                 /* update to the next state and also initialize TCB */
 139                 tx_info->connection_state = new_state;
 140                 /* FALLTHRU */
 141         case KTLS_CONN_ACT_OPEN_RPL:
 142                 /* if we are stuck in this state, means tcb init might not
 143                  * received by HW, try sending it again.
 144                  */
 145                 if (!chcr_init_tcb_fields(tx_info))
 146                         tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
 147                 break;
 148
 149         case KTLS_CONN_SET_TCB_REQ:
 150                 /* only go forward if state is greater than current state. */
 151                 if (new_state <= tx_info->connection_state)
 152                         break;
 153                 /* update to the next state and check if l2t_state is valid  */
 154                 tx_info->connection_state = new_state;
 155                 /* FALLTHRU */
 156         case KTLS_CONN_SET_TCB_RPL:
 157                 /* Check if l2t state is valid, then move to ready state. */
 158                 if (cxgb4_check_l2t_valid(tx_info->l2te)) {
 159                         tx_info->connection_state = KTLS_CONN_TX_READY;
 160                         atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx);
 161                 }
 162                 break;
 163
 164         case KTLS_CONN_TX_READY:
 165                 /* nothing to be done here */
 166                 break;
 167
 168         default:
 169                 pr_err("unknown KTLS connection state\n");
 170                 break;
 171         }
 172         spin_unlock_irqrestore(&tx_info->lock, flags);
 173
 174         return tx_info->connection_state;
 175 }
 176 /*
 177  * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
 178  * @sk - tcp socket.
 179  * @tx_info - driver specific tls info.
 180  * @atid - connection active tid.
 181  * return - send success/failure.
 182  */
 183 static int chcr_ktls_act_open_req(struct sock *sk,
 184                                   struct chcr_ktls_info *tx_info,
 185                                   int atid)
 186 {
 187         struct inet_sock *inet = inet_sk(sk);
 188         struct cpl_t6_act_open_req *cpl6;
 189         struct cpl_act_open_req *cpl;
 190         struct sk_buff *skb;
 191         unsigned int len;
 192         int qid_atid;
 193         u64 options;
 194
 195         len = sizeof(*cpl6);
 196         skb = alloc_skb(len, GFP_KERNEL);
 197         if (unlikely(!skb))
 198                 return -ENOMEM;
 199         /* mark it a control pkt */
 200         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 201
 202         cpl6 = __skb_put_zero(skb, len);
 203         cpl = (struct cpl_act_open_req *)cpl6;
 204         INIT_TP_WR(cpl6, 0);
 205         qid_atid = TID_QID_V(tx_info->rx_qid) |
 206                    TID_TID_V(atid);
 207         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
 208         cpl->local_port = inet->inet_sport;
 209         cpl->peer_port = inet->inet_dport;
 210         cpl->local_ip = inet->inet_rcv_saddr;
 211         cpl->peer_ip = inet->inet_daddr;
 212
 213         /* fill first 64 bit option field. */
 214         options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 215                   SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 216         cpl->opt0 = cpu_to_be64(options);
 217
 218         /* next 64 bit option field. */
 219         options =
 220                 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 221         cpl->opt2 = htonl(options);
 222
 223         return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 224 }
 225
 226 /*
 227  * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
 228  * @sk - tcp socket.
 229  * @tx_info - driver specific tls info.
 230  * @atid - connection active tid.
 231  * return - send success/failure.
 232  */
 233 static int chcr_ktls_act_open_req6(struct sock *sk,
 234                                    struct chcr_ktls_info *tx_info,
 235                                    int atid)
 236 {
 237         struct inet_sock *inet = inet_sk(sk);
 238         struct cpl_t6_act_open_req6 *cpl6;
 239         struct cpl_act_open_req6 *cpl;
 240         struct sk_buff *skb;
 241         unsigned int len;
 242         int qid_atid;
 243         u64 options;
 244
 245         len = sizeof(*cpl6);
 246         skb = alloc_skb(len, GFP_KERNEL);
 247         if (unlikely(!skb))
 248                 return -ENOMEM;
 249         /* mark it a control pkt */
 250         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 251
 252         cpl6 = __skb_put_zero(skb, len);
 253         cpl = (struct cpl_act_open_req6 *)cpl6;
 254         INIT_TP_WR(cpl6, 0);
 255         qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
 256         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
 257         cpl->local_port = inet->inet_sport;
 258         cpl->peer_port = inet->inet_dport;
 259         cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
 260         cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
 261         cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
 262         cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
 263
 264         /* first 64 bit option field. */
 265         options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 266                   SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 267         cpl->opt0 = cpu_to_be64(options);
 268         /* next 64 bit option field. */
 269         options =
 270                 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 271         cpl->opt2 = htonl(options);
 272
 273         return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 274 }
 275
 276 /*
 277  * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
 278  * @sk - tcp socket.
 279  * @tx_info - driver specific tls info.
 280  * return: NET_TX_OK/NET_XMIT_DROP
 281  */
 282 static int chcr_setup_connection(struct sock *sk,
 283                                  struct chcr_ktls_info *tx_info)
 284 {
 285         struct tid_info *t = &tx_info->adap->tids;
 286         int atid, ret = 0;
 287
 288         atid = cxgb4_alloc_atid(t, tx_info);
 289         if (atid == -1)
 290                 return -EINVAL;
 291
 292         tx_info->atid = atid;
 293         tx_info->ip_family = sk->sk_family;
 294
 295         if (sk->sk_family == AF_INET ||
 296             (sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
 297              ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
 298                 tx_info->ip_family = AF_INET;
 299                 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
 300         } else {
 301                 tx_info->ip_family = AF_INET6;
 302                 ret =
 303                 cxgb4_clip_get(tx_info->netdev,
 304                                (const u32 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8,
 305                                1);
 306                 if (ret)
 307                         goto out;
 308                 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
 309         }
 310
 311         /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
 312          * success, if any other return type clear atid and return that failure.
 313          */
 314         if (ret) {
 315                 if (ret == NET_XMIT_CN)
 316                         ret = 0;
 317                 else
 318                         cxgb4_free_atid(t, atid);
 319                 goto out;
 320         }
 321
 322         /* update the connection state */
 323         chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
 324 out:
 325         return ret;
 326 }
 327
 328 /*
 329  * chcr_set_tcb_field: update tcb fields.
 330  * @tx_info - driver specific tls info.
 331  * @word - TCB word.
 332  * @mask - TCB word related mask.
 333  * @val - TCB word related value.
 334  * @no_reply - set 1 if not looking for TP response.
 335  */
 336 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
 337                               u64 mask, u64 val, int no_reply)
 338 {
 339         struct cpl_set_tcb_field *req;
 340         struct sk_buff *skb;
 341
 342         skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
 343         if (!skb)
 344                 return -ENOMEM;
 345
 346         req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
 347         INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
 348         req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 349                                 NO_REPLY_V(no_reply));
 350         req->word_cookie = htons(TCB_WORD_V(word));
 351         req->mask = cpu_to_be64(mask);
 352         req->val = cpu_to_be64(val);
 353
 354         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 355         return cxgb4_ofld_send(tx_info->netdev, skb);
 356 }
 357
 358 /*
 359  * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
 360  * @tx_info - driver specific tls info.
 361  * return: NET_TX_OK/NET_XMIT_DROP.
 362  */
 363 static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
 364 {
 365         return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
 366                                   TCB_T_STATE_V(TCB_T_STATE_M),
 367                                   CHCR_TCB_STATE_CLOSED, 1);
 368 }
 369
 370 /*
 371  * chcr_ktls_dev_del:  call back for tls_dev_del.
 372  * Remove the tid and l2t entry and close the connection.
 373  * it per connection basis.
 374  * @netdev - net device.
 375  * @tls_cts - tls context.
 376  * @direction - TX/RX crypto direction
 377  */
 378 static void chcr_ktls_dev_del(struct net_device *netdev,
 379                               struct tls_context *tls_ctx,
 380                               enum tls_offload_ctx_dir direction)
 381 {
 382         struct chcr_ktls_ofld_ctx_tx *tx_ctx =
 383                                 chcr_get_ktls_tx_context(tls_ctx);
 384         struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
 385         struct sock *sk;
 386
 387         if (!tx_info)
 388                 return;
 389         sk = tx_info->sk;
 390
 391         spin_lock(&tx_info->lock);
 392         tx_info->connection_state = KTLS_CONN_CLOSED;
 393         spin_unlock(&tx_info->lock);
 394
 395         /* clear l2t entry */
 396         if (tx_info->l2te)
 397                 cxgb4_l2t_release(tx_info->l2te);
 398
 399         /* clear clip entry */
 400         if (tx_info->ip_family == AF_INET6)
 401                 cxgb4_clip_release(netdev,
 402                                    (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8,
 403                                    1);
 404
 405         /* clear tid */
 406         if (tx_info->tid != -1) {
 407                 /* clear tcb state and then release tid */
 408                 chcr_ktls_mark_tcb_close(tx_info);
 409                 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 410                                  tx_info->tid, tx_info->ip_family);
 411         }
 412
 413         atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close);
 414         kvfree(tx_info);
 415         tx_ctx->chcr_info = NULL;
 416 }
 417
 418 /*
 419  * chcr_ktls_dev_add:  call back for tls_dev_add.
 420  * Create a tcb entry for TP. Also add l2t entry for the connection. And
 421  * generate keys & save those keys locally.
 422  * @netdev - net device.
 423  * @tls_cts - tls context.
 424  * @direction - TX/RX crypto direction
 425  * return: SUCCESS/FAILURE.
 426  */
 427 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
 428                              enum tls_offload_ctx_dir direction,
 429                              struct tls_crypto_info *crypto_info,
 430                              u32 start_offload_tcp_sn)
 431 {
 432         struct tls_context *tls_ctx = tls_get_ctx(sk);
 433         struct chcr_ktls_ofld_ctx_tx *tx_ctx;
 434         struct chcr_ktls_info *tx_info;
 435         struct dst_entry *dst;
 436         struct adapter *adap;
 437         struct port_info *pi;
 438         struct neighbour *n;
 439         u8 daaddr[16];
 440         int ret = -1;
 441
 442         tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
 443
 444         pi = netdev_priv(netdev);
 445         adap = pi->adapter;
 446         if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
 447                 pr_err("not expecting for RX direction\n");
 448                 ret = -EINVAL;
 449                 goto out;
 450         }
 451         if (tx_ctx->chcr_info) {
 452                 ret = -EINVAL;
 453                 goto out;
 454         }
 455
 456         tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
 457         if (!tx_info) {
 458                 ret = -ENOMEM;
 459                 goto out;
 460         }
 461
 462         spin_lock_init(&tx_info->lock);
 463
 464         /* clear connection state */
 465         spin_lock(&tx_info->lock);
 466         tx_info->connection_state = KTLS_CONN_CLOSED;
 467         spin_unlock(&tx_info->lock);
 468
 469         tx_info->sk = sk;
 470         /* initialize tid and atid to -1, 0 is a also a valid id. */
 471         tx_info->tid = -1;
 472         tx_info->atid = -1;
 473
 474         tx_info->adap = adap;
 475         tx_info->netdev = netdev;
 476         tx_info->first_qset = pi->first_qset;
 477         tx_info->tx_chan = pi->tx_chan;
 478         tx_info->smt_idx = pi->smt_idx;
 479         tx_info->port_id = pi->port_id;
 480
 481         tx_info->rx_qid = chcr_get_first_rx_qid(adap);
 482         if (unlikely(tx_info->rx_qid < 0))
 483                 goto out2;
 484
 485         tx_info->prev_seq = start_offload_tcp_sn;
 486         tx_info->tcp_start_seq_number = start_offload_tcp_sn;
 487
 488         /* save crypto keys */
 489         ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
 490         if (ret < 0)
 491                 goto out2;
 492
 493         /* get peer ip */
 494         if (sk->sk_family == AF_INET ||
 495             (sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
 496              ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
 497                 memcpy(daaddr, &sk->sk_daddr, 4);
 498         } else {
 499                 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
 500         }
 501
 502         /* get the l2t index */
 503         dst = sk_dst_get(sk);
 504         if (!dst) {
 505                 pr_err("DST entry not found\n");
 506                 goto out2;
 507         }
 508         n = dst_neigh_lookup(dst, daaddr);
 509         if (!n || !n->dev) {
 510                 pr_err("neighbour not found\n");
 511                 dst_release(dst);
 512                 goto out2;
 513         }
 514         tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
 515
 516         neigh_release(n);
 517         dst_release(dst);
 518
 519         if (!tx_info->l2te) {
 520                 pr_err("l2t entry not found\n");
 521                 goto out2;
 522         }
 523
 524         tx_ctx->chcr_info = tx_info;
 525
 526         /* create a filter and call cxgb4_l2t_send to send the packet out, which
 527          * will take care of updating l2t entry in hw if not already done.
 528          */
 529         ret = chcr_setup_connection(sk, tx_info);
 530         if (ret)
 531                 goto out2;
 532
 533         atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open);
 534         return 0;
 535 out2:
 536         kvfree(tx_info);
 537 out:
 538         atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail);
 539         return ret;
 540 }
 541
 542 static const struct tlsdev_ops chcr_ktls_ops = {
 543         .tls_dev_add = chcr_ktls_dev_add,
 544         .tls_dev_del = chcr_ktls_dev_del,
 545 };
 546
 547 /*
 548  * chcr_enable_ktls:  add NETIF_F_HW_TLS_TX flag in all the ports.
 549  */
 550 void chcr_enable_ktls(struct adapter *adap)
 551 {
 552         struct net_device *netdev;
 553         int i;
 554
 555         for_each_port(adap, i) {
 556                 netdev = adap->port[i];
 557                 netdev->features |= NETIF_F_HW_TLS_TX;
 558                 netdev->hw_features |= NETIF_F_HW_TLS_TX;
 559                 netdev->tlsdev_ops = &chcr_ktls_ops;
 560         }
 561 }
 562
 563 /*
 564  * chcr_disable_ktls:  remove NETIF_F_HW_TLS_TX flag from all the ports.
 565  */
 566 void chcr_disable_ktls(struct adapter *adap)
 567 {
 568         struct net_device *netdev;
 569         int i;
 570
 571         for_each_port(adap, i) {
 572                 netdev = adap->port[i];
 573                 netdev->features &= ~NETIF_F_HW_TLS_TX;
 574                 netdev->hw_features &= ~NETIF_F_HW_TLS_TX;
 575                 netdev->tlsdev_ops = NULL;
 576         }
 577 }
 578
 579 /*
 580  * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
 581  *                        handling.
 582  * @tx_info - driver specific tls info.
 583  * return: NET_TX_OK/NET_XMIT_DROP
 584  */
 585 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
 586 {
 587         int  ret = 0;
 588
 589         /* set tcb in offload and bypass */
 590         ret =
 591         chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
 592                            TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
 593                            TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
 594         if (ret)
 595                 return ret;
 596         /* reset snd_una and snd_next fields in tcb */
 597         ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
 598                                  TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
 599                                  TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
 600                                  0, 1);
 601         if (ret)
 602                 return ret;
 603
 604         /* reset send max */
 605         ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
 606                                  TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
 607                                  0, 1);
 608         if (ret)
 609                 return ret;
 610
 611         /* update l2t index and request for tp reply to confirm tcb is
 612          * initialised to handle tx traffic.
 613          */
 614         ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
 615                                  TCB_L2T_IX_V(TCB_L2T_IX_M),
 616                                  TCB_L2T_IX_V(tx_info->l2te->idx), 0);
 617         return ret;
 618 }
 619
 620 /*
 621  * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
 622  */
 623 int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input)
 624 {
 625         const struct cpl_act_open_rpl *p = (void *)input;
 626         struct chcr_ktls_info *tx_info = NULL;
 627         unsigned int atid, tid, status;
 628         struct tid_info *t;
 629
 630         tid = GET_TID(p);
 631         status = AOPEN_STATUS_G(ntohl(p->atid_status));
 632         atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
 633
 634         t = &adap->tids;
 635         tx_info = lookup_atid(t, atid);
 636
 637         if (!tx_info || tx_info->atid != atid) {
 638                 pr_err("tx_info or atid is not correct\n");
 639                 return -1;
 640         }
 641
 642         if (!status) {
 643                 tx_info->tid = tid;
 644                 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
 645
 646                 cxgb4_free_atid(t, atid);
 647                 tx_info->atid = -1;
 648                 /* update the connection state */
 649                 chcr_ktls_update_connection_state(tx_info,
 650                                                   KTLS_CONN_ACT_OPEN_RPL);
 651         }
 652         return 0;
 653 }
 654
 655 /*
 656  * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
 657  */
 658 int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
 659 {
 660         const struct cpl_set_tcb_rpl *p = (void *)input;
 661         struct chcr_ktls_info *tx_info = NULL;
 662         struct tid_info *t;
 663         u32 tid;
 664
 665         tid = GET_TID(p);
 666
 667         t = &adap->tids;
 668         tx_info = lookup_tid(t, tid);
 669         if (!tx_info || tx_info->tid != tid) {
 670                 pr_err("tx_info or atid is not correct\n");
 671                 return -1;
 672         }
 673         /* update the connection state */
 674         chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
 675         return 0;
 676 }
 677
 678 /*
 679  * chcr_write_cpl_set_tcb_ulp: update tcb values.
 680  * TCB is responsible to create tcp headers, so all the related values
 681  * should be correctly updated.
 682  * @tx_info - driver specific tls info.
 683  * @q - tx queue on which packet is going out.
 684  * @tid - TCB identifier.
 685  * @pos - current index where should we start writing.
 686  * @word - TCB word.
 687  * @mask - TCB word related mask.
 688  * @val - TCB word related value.
 689  * @reply - set 1 if looking for TP response.
 690  * return - next position to write.
 691  */
 692 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 693                                         struct sge_eth_txq *q, u32 tid,
 694                                         void *pos, u16 word, u64 mask,
 695                                         u64 val, u32 reply)
 696 {
 697         struct cpl_set_tcb_field_core *cpl;
 698         struct ulptx_idata *idata;
 699         struct ulp_txpkt *txpkt;
 700         void *save_pos = NULL;
 701         u8 buf[48] = {0};
 702         int left;
 703
 704         left = (void *)q->q.stat - pos;
 705         if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
 706                 if (!left) {
 707                         pos = q->q.desc;
 708                 } else {
 709                         save_pos = pos;
 710                         pos = buf;
 711                 }
 712         }
 713         /* ULP_TXPKT */
 714         txpkt = pos;
 715         txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0));
 716         txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
 717
 718         /* ULPTX_IDATA sub-command */
 719         idata = (struct ulptx_idata *)(txpkt + 1);
 720         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
 721         idata->len = htonl(sizeof(*cpl));
 722         pos = idata + 1;
 723
 724         cpl = pos;
 725         /* CPL_SET_TCB_FIELD */
 726         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 727         cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 728                         NO_REPLY_V(!reply));
 729         cpl->word_cookie = htons(TCB_WORD_V(word));
 730         cpl->mask = cpu_to_be64(mask);
 731         cpl->val = cpu_to_be64(val);
 732
 733         /* ULPTX_NOOP */
 734         idata = (struct ulptx_idata *)(cpl + 1);
 735         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
 736         idata->len = htonl(0);
 737
 738         if (save_pos) {
 739                 pos = chcr_copy_to_txd(buf, &q->q, save_pos,
 740                                        CHCR_SET_TCB_FIELD_LEN);
 741         } else {
 742                 /* check again if we are at the end of the queue */
 743                 if (left == CHCR_SET_TCB_FIELD_LEN)
 744                         pos = q->q.desc;
 745                 else
 746                         pos = idata + 1;
 747         }
 748
 749         return pos;
 750 }
 751
 752 /*
 753  * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
 754  * with updated values like tcp seq, ack, window etc.
 755  * @tx_info - driver specific tls info.
 756  * @q - TX queue.
 757  * @tcp_seq
 758  * @tcp_ack
 759  * @tcp_win
 760  * return: NETDEV_TX_BUSY/NET_TX_OK.
 761  */
 762 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
 763                                    struct sge_eth_txq *q, u64 tcp_seq,
 764                                    u64 tcp_ack, u64 tcp_win)
 765 {
 766         bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
 767         u32 len, cpl = 0, ndesc, wr_len;
 768         struct fw_ulptx_wr *wr;
 769         int credits;
 770         void *pos;
 771
 772         wr_len = sizeof(*wr);
 773         /* there can be max 4 cpls, check if we have enough credits */
 774         len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 775         ndesc = DIV_ROUND_UP(len, 64);
 776
 777         credits = chcr_txq_avail(&q->q) - ndesc;
 778         if (unlikely(credits < 0)) {
 779                 chcr_eth_txq_stop(q);
 780                 return NETDEV_TX_BUSY;
 781         }
 782
 783         pos = &q->q.desc[q->q.pidx];
 784         /* make space for WR, we'll fill it later when we know all the cpls
 785          * being sent out and have complete length.
 786          */
 787         wr = pos;
 788         pos += wr_len;
 789         /* update tx_max if its a re-transmit or the first wr */
 790         if (first_wr || tcp_seq != tx_info->prev_seq) {
 791                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 792                                                  TCB_TX_MAX_W,
 793                                                  TCB_TX_MAX_V(TCB_TX_MAX_M),
 794                                                  TCB_TX_MAX_V(tcp_seq), 0);
 795                 cpl++;
 796         }
 797         /* reset snd una if it's a re-transmit pkt */
 798         if (tcp_seq != tx_info->prev_seq) {
 799                 /* reset snd_una */
 800                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 801                                                  TCB_SND_UNA_RAW_W,
 802                                                  TCB_SND_UNA_RAW_V
 803                                                  (TCB_SND_UNA_RAW_M),
 804                                                  TCB_SND_UNA_RAW_V(0), 0);
 805                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo);
 806                 cpl++;
 807         }
 808         /* update ack */
 809         if (first_wr || tx_info->prev_ack != tcp_ack) {
 810                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 811                                                  TCB_RCV_NXT_W,
 812                                                  TCB_RCV_NXT_V(TCB_RCV_NXT_M),
 813                                                  TCB_RCV_NXT_V(tcp_ack), 0);
 814                 tx_info->prev_ack = tcp_ack;
 815                 cpl++;
 816         }
 817         /* update receive window */
 818         if (first_wr || tx_info->prev_win != tcp_win) {
 819                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 820                                                  TCB_RCV_WND_W,
 821                                                  TCB_RCV_WND_V(TCB_RCV_WND_M),
 822                                                  TCB_RCV_WND_V(tcp_win), 0);
 823                 tx_info->prev_win = tcp_win;
 824                 cpl++;
 825         }
 826
 827         if (cpl) {
 828                 /* get the actual length */
 829                 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 830                 /* ULPTX wr */
 831                 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
 832                 wr->cookie = 0;
 833                 /* fill len in wr field */
 834                 wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
 835
 836                 ndesc = DIV_ROUND_UP(len, 64);
 837                 chcr_txq_advance(&q->q, ndesc);
 838                 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
 839         }
 840         return 0;
 841 }
 842
 843 /*
 844  * chcr_ktls_skb_copy
 845  * @nskb - new skb where the frags to be added.
 846  * @skb - old skb from which frags will be copied.
 847  */
 848 static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb)
 849 {
 850         int i;
 851
 852         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 853                 skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i];
 854                 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
 855         }
 856
 857         skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags;
 858         nskb->len += skb->data_len;
 859         nskb->data_len = skb->data_len;
 860         nskb->truesize += skb->data_len;
 861 }
 862
 863 /*
 864  * chcr_ktls_get_tx_flits
 865  * returns number of flits to be sent out, it includes key context length, WR
 866  * size and skb fragments.
 867  */
 868 static unsigned int
 869 chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len)
 870 {
 871         return chcr_sgl_len(skb_shinfo(skb)->nr_frags) +
 872                DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
 873 }
 874
 875 /*
 876  * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
 877  * other than timestamp.
 878  * @skb - skb contains partial record..
 879  * return: 1 / 0
 880  */
 881 static int
 882 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
 883 {
 884         int cnt, opt, optlen;
 885         u_char *cp;
 886
 887         cp = (u_char *)(tcp + 1);
 888         cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
 889         for (; cnt > 0; cnt -= optlen, cp += optlen) {
 890                 opt = cp[0];
 891                 if (opt == TCPOPT_EOL)
 892                         break;
 893                 if (opt == TCPOPT_NOP) {
 894                         optlen = 1;
 895                 } else {
 896                         if (cnt < 2)
 897                                 break;
 898                         optlen = cp[1];
 899                         if (optlen < 2 || optlen > cnt)
 900                                 break;
 901                 }
 902                 switch (opt) {
 903                 case TCPOPT_NOP:
 904                         break;
 905                 default:
 906                         return 1;
 907                 }
 908         }
 909         return 0;
 910 }
 911
 912 /*
 913  * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
 914  * send out separately.
 915  * @tx_info - driver specific tls info.
 916  * @skb - skb contains partial record..
 917  * @q - TX queue.
 918  * @tx_chan - channel number.
 919  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
 920  */
 921 static int
 922 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
 923                             struct sge_eth_txq *q, uint32_t tx_chan)
 924 {
 925         struct fw_eth_tx_pkt_wr *wr;
 926         struct cpl_tx_pkt_core *cpl;
 927         u32 ctrl, iplen, maclen;
 928         struct ipv6hdr *ip6;
 929         unsigned int ndesc;
 930         struct tcphdr *tcp;
 931         int len16, pktlen;
 932         struct iphdr *ip;
 933         int credits;
 934         u8 buf[150];
 935         void *pos;
 936
 937         iplen = skb_network_header_len(skb);
 938         maclen = skb_mac_header_len(skb);
 939
 940         /* packet length = eth hdr len + ip hdr len + tcp hdr len
 941          * (including options).
 942          */
 943         pktlen = skb->len - skb->data_len;
 944
 945         ctrl = sizeof(*cpl) + pktlen;
 946         len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
 947         /* check how many descriptors needed */
 948         ndesc = DIV_ROUND_UP(len16, 4);
 949
 950         credits = chcr_txq_avail(&q->q) - ndesc;
 951         if (unlikely(credits < 0)) {
 952                 chcr_eth_txq_stop(q);
 953                 return NETDEV_TX_BUSY;
 954         }
 955
 956         pos = &q->q.desc[q->q.pidx];
 957         wr = pos;
 958
 959         /* Firmware work request header */
 960         wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
 961                                FW_WR_IMMDLEN_V(ctrl));
 962
 963         wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16));
 964         wr->r3 = 0;
 965
 966         cpl = (void *)(wr + 1);
 967
 968         /* CPL header */
 969         cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
 970                            TXPKT_PF_V(tx_info->adap->pf));
 971         cpl->pack = 0;
 972         cpl->len = htons(pktlen);
 973         /* checksum offload */
 974         cpl->ctrl1 = 0;
 975
 976         pos = cpl + 1;
 977
 978         memcpy(buf, skb->data, pktlen);
 979         if (tx_info->ip_family == AF_INET) {
 980                 /* we need to correct ip header len */
 981                 ip = (struct iphdr *)(buf + maclen);
 982                 ip->tot_len = htons(pktlen - maclen);
 983         } else {
 984                 ip6 = (struct ipv6hdr *)(buf + maclen);
 985                 ip6->payload_len = htons(pktlen - maclen - iplen);
 986         }
 987         /* now take care of the tcp header, if fin is not set then clear push
 988          * bit as well, and if fin is set, it will be sent at the last so we
 989          * need to update the tcp sequence number as per the last packet.
 990          */
 991         tcp = (struct tcphdr *)(buf + maclen + iplen);
 992
 993         if (!tcp->fin)
 994                 tcp->psh = 0;
 995         else
 996                 tcp->seq = htonl(tx_info->prev_seq);
 997
 998         chcr_copy_to_txd(buf, &q->q, pos, pktlen);
 999
1000         chcr_txq_advance(&q->q, ndesc);
1001         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1002         return 0;
1003 }
1004
1005 /* chcr_ktls_skb_shift - Shifts request length paged data from skb to another.
1006  * @tgt- buffer into which tail data gets added
1007  * @skb- buffer from which the paged data comes from
1008  * @shiftlen- shift up to this many bytes
1009  */
1010 static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
1011                                int shiftlen)
1012 {
1013         skb_frag_t *fragfrom, *fragto;
1014         int from, to, todo;
1015
1016         WARN_ON(shiftlen > skb->data_len);
1017
1018         todo = shiftlen;
1019         from = 0;
1020         to = 0;
1021         fragfrom = &skb_shinfo(skb)->frags[from];
1022
1023         while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
1024                 fragfrom = &skb_shinfo(skb)->frags[from];
1025                 fragto = &skb_shinfo(tgt)->frags[to];
1026
1027                 if (todo >= skb_frag_size(fragfrom)) {
1028                         *fragto = *fragfrom;
1029                         todo -= skb_frag_size(fragfrom);
1030                         from++;
1031                         to++;
1032
1033                 } else {
1034                         __skb_frag_ref(fragfrom);
1035                         skb_frag_page_copy(fragto, fragfrom);
1036                         skb_frag_off_copy(fragto, fragfrom);
1037                         skb_frag_size_set(fragto, todo);
1038
1039                         skb_frag_off_add(fragfrom, todo);
1040                         skb_frag_size_sub(fragfrom, todo);
1041                         todo = 0;
1042
1043                         to++;
1044                         break;
1045                 }
1046         }
1047
1048         /* Ready to "commit" this state change to tgt */
1049         skb_shinfo(tgt)->nr_frags = to;
1050
1051         /* Reposition in the original skb */
1052         to = 0;
1053         while (from < skb_shinfo(skb)->nr_frags)
1054                 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
1055
1056         skb_shinfo(skb)->nr_frags = to;
1057
1058         WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
1059
1060         skb->len -= shiftlen;
1061         skb->data_len -= shiftlen;
1062         skb->truesize -= shiftlen;
1063         tgt->len += shiftlen;
1064         tgt->data_len += shiftlen;
1065         tgt->truesize += shiftlen;
1066
1067         return shiftlen;
1068 }
1069
1070 /*
1071  * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1072  * received has partial end part of the record, send out the complete record, so
1073  * that crypto block will be able to generate TAG/HASH.
1074  * @skb - segment which has complete or partial end part.
1075  * @tx_info - driver specific tls info.
1076  * @q - TX queue.
1077  * @tcp_seq
1078  * @tcp_push - tcp push bit.
1079  * @mss - segment size.
1080  * return: NETDEV_TX_BUSY/NET_TX_OK.
1081  */
1082 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1083                                       struct chcr_ktls_info *tx_info,
1084                                       struct sge_eth_txq *q, u32 tcp_seq,
1085                                       bool tcp_push, u32 mss)
1086 {
1087         u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1088         struct adapter *adap = tx_info->adap;
1089         int credits, left, last_desc;
1090         struct tx_sw_desc *sgl_sdesc;
1091         struct cpl_tx_data *tx_data;
1092         struct cpl_tx_sec_pdu *cpl;
1093         struct ulptx_idata *idata;
1094         struct ulp_txpkt *ulptx;
1095         struct fw_ulptx_wr *wr;
1096         void *pos;
1097         u64 *end;
1098
1099         /* get the number of flits required */
1100         flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len);
1101         /* number of descriptors */
1102         ndesc = chcr_flits_to_desc(flits);
1103         /* check if enough credits available */
1104         credits = chcr_txq_avail(&q->q) - ndesc;
1105         if (unlikely(credits < 0)) {
1106                 chcr_eth_txq_stop(q);
1107                 return NETDEV_TX_BUSY;
1108         }
1109
1110         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1111                 /* Credits are below the threshold vaues, stop the queue after
1112                  * injecting the Work Request for this packet.
1113                  */
1114                 chcr_eth_txq_stop(q);
1115                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1116         }
1117
1118         last_desc = q->q.pidx + ndesc - 1;
1119         if (last_desc >= q->q.size)
1120                 last_desc -= q->q.size;
1121         sgl_sdesc = &q->q.sdesc[last_desc];
1122
1123         if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1124                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1125                 q->mapping_err++;
1126                 return NETDEV_TX_BUSY;
1127         }
1128
1129         pos = &q->q.desc[q->q.pidx];
1130         end = (u64 *)pos + flits;
1131         /* FW_ULPTX_WR */
1132         wr = pos;
1133         /* WR will need len16 */
1134         len16 = DIV_ROUND_UP(flits, 2);
1135         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1136         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1137         wr->cookie = 0;
1138         pos += sizeof(*wr);
1139         /* ULP_TXPKT */
1140         ulptx = pos;
1141         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1142                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1143                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1144                                 ULP_TXPKT_RO_F);
1145         ulptx->len = htonl(len16 - 1);
1146         /* ULPTX_IDATA sub-command */
1147         idata = (struct ulptx_idata *)(ulptx + 1);
1148         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1149         /* idata length will include cpl_tx_sec_pdu + key context size +
1150          * cpl_tx_data header.
1151          */
1152         idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1153                            sizeof(*tx_data));
1154         /* SEC CPL */
1155         cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1156         cpl->op_ivinsrtofst =
1157                 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1158                       CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1159                       CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1160                       CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1161         cpl->pldlen = htonl(skb->data_len);
1162
1163         /* encryption should start after tls header size + iv size */
1164         cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1165
1166         cpl->aadstart_cipherstop_hi =
1167                 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1168                       CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1169                       CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1170
1171         /* authentication will also start after tls header + iv size */
1172         cpl->cipherstop_lo_authinsert =
1173         htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1174               CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1175               CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1176
1177         /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1178         cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1179         cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1180         cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1181
1182         pos = cpl + 1;
1183         /* check if space left to fill the keys */
1184         left = (void *)q->q.stat - pos;
1185         if (!left) {
1186                 left = (void *)end - (void *)q->q.stat;
1187                 pos = q->q.desc;
1188                 end = pos + left;
1189         }
1190
1191         pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1192                                tx_info->key_ctx_len);
1193         left = (void *)q->q.stat - pos;
1194
1195         if (!left) {
1196                 left = (void *)end - (void *)q->q.stat;
1197                 pos = q->q.desc;
1198                 end = pos + left;
1199         }
1200         /* CPL_TX_DATA */
1201         tx_data = (void *)pos;
1202         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1203         tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len));
1204
1205         tx_data->rsvd = htonl(tcp_seq);
1206
1207         tx_data->flags = htonl(TX_BYPASS_F);
1208         if (tcp_push)
1209                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1210
1211         /* check left again, it might go beyond queue limit */
1212         pos = tx_data + 1;
1213         left = (void *)q->q.stat - pos;
1214
1215         /* check the position again */
1216         if (!left) {
1217                 left = (void *)end - (void *)q->q.stat;
1218                 pos = q->q.desc;
1219                 end = pos + left;
1220         }
1221
1222         /* send the complete packet except the header */
1223         cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1224                         sgl_sdesc->addr);
1225         sgl_sdesc->skb = skb;
1226
1227         chcr_txq_advance(&q->q, ndesc);
1228         cxgb4_ring_tx_db(adap, &q->q, ndesc);
1229         atomic64_inc(&adap->chcr_stats.ktls_tx_send_records);
1230
1231         return 0;
1232 }
1233
1234 /*
1235  * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1236  * a middle part of a record, fetch the prior data to make it 16 byte aligned
1237  * and then only send it out.
1238  *
1239  * @skb - skb contains partial record..
1240  * @tx_info - driver specific tls info.
1241  * @q - TX queue.
1242  * @tcp_seq
1243  * @tcp_push - tcp push bit.
1244  * @mss - segment size.
1245  * @tls_rec_offset - offset from start of the tls record.
1246  * @perior_data - data before the current segment, required to make this record
1247  *                16 byte aligned.
1248  * @prior_data_len - prior_data length (less than 16)
1249  * return: NETDEV_TX_BUSY/NET_TX_OK.
1250  */
1251 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1252                                    struct chcr_ktls_info *tx_info,
1253                                    struct sge_eth_txq *q,
1254                                    u32 tcp_seq, bool tcp_push, u32 mss,
1255                                    u32 tls_rec_offset, u8 *prior_data,
1256                                    u32 prior_data_len)
1257 {
1258         struct adapter *adap = tx_info->adap;
1259         u32 len16, wr_mid = 0, cipher_start;
1260         unsigned int flits = 0, ndesc;
1261         int credits, left, last_desc;
1262         struct tx_sw_desc *sgl_sdesc;
1263         struct cpl_tx_data *tx_data;
1264         struct cpl_tx_sec_pdu *cpl;
1265         struct ulptx_idata *idata;
1266         struct ulp_txpkt *ulptx;
1267         struct fw_ulptx_wr *wr;
1268         __be64 iv_record;
1269         void *pos;
1270         u64 *end;
1271
1272         /* get the number of flits required, it's a partial record so 2 flits
1273          * (AES_BLOCK_SIZE) will be added.
1274          */
1275         flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2;
1276         /* get the correct 8 byte IV of this record */
1277         iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1278         /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1279          * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1280          * data will be added.
1281          */
1282         if (prior_data_len)
1283                 flits += 2;
1284         /* number of descriptors */
1285         ndesc = chcr_flits_to_desc(flits);
1286         /* check if enough credits available */
1287         credits = chcr_txq_avail(&q->q) - ndesc;
1288         if (unlikely(credits < 0)) {
1289                 chcr_eth_txq_stop(q);
1290                 return NETDEV_TX_BUSY;
1291         }
1292
1293         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1294                 chcr_eth_txq_stop(q);
1295                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1296         }
1297
1298         last_desc = q->q.pidx + ndesc - 1;
1299         if (last_desc >= q->q.size)
1300                 last_desc -= q->q.size;
1301         sgl_sdesc = &q->q.sdesc[last_desc];
1302
1303         if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1304                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1305                 q->mapping_err++;
1306                 return NETDEV_TX_BUSY;
1307         }
1308
1309         pos = &q->q.desc[q->q.pidx];
1310         end = (u64 *)pos + flits;
1311         /* FW_ULPTX_WR */
1312         wr = pos;
1313         /* WR will need len16 */
1314         len16 = DIV_ROUND_UP(flits, 2);
1315         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1316         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1317         wr->cookie = 0;
1318         pos += sizeof(*wr);
1319         /* ULP_TXPKT */
1320         ulptx = pos;
1321         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1322                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1323                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1324                                 ULP_TXPKT_RO_F);
1325         ulptx->len = htonl(len16 - 1);
1326         /* ULPTX_IDATA sub-command */
1327         idata = (struct ulptx_idata *)(ulptx + 1);
1328         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1329         /* idata length will include cpl_tx_sec_pdu + key context size +
1330          * cpl_tx_data header.
1331          */
1332         idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1333                            sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1334         /* SEC CPL */
1335         cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1336         /* cipher start will have tls header + iv size extra if its a header
1337          * part of tls record. else only 16 byte IV will be added.
1338          */
1339         cipher_start =
1340                 AES_BLOCK_LEN + 1 +
1341                 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1342
1343         cpl->op_ivinsrtofst =
1344                 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1345                       CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1346                       CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1347         cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len);
1348         cpl->aadstart_cipherstop_hi =
1349                 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1350         cpl->cipherstop_lo_authinsert = 0;
1351         /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1352         cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1353         cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1354         cpl->scmd1 = 0;
1355
1356         pos = cpl + 1;
1357         /* check if space left to fill the keys */
1358         left = (void *)q->q.stat - pos;
1359         if (!left) {
1360                 left = (void *)end - (void *)q->q.stat;
1361                 pos = q->q.desc;
1362                 end = pos + left;
1363         }
1364
1365         pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1366                                tx_info->key_ctx_len);
1367         left = (void *)q->q.stat - pos;
1368
1369         if (!left) {
1370                 left = (void *)end - (void *)q->q.stat;
1371                 pos = q->q.desc;
1372                 end = pos + left;
1373         }
1374         /* CPL_TX_DATA */
1375         tx_data = (void *)pos;
1376         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1377         tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1378                         TX_LENGTH_V(skb->data_len + prior_data_len));
1379         tx_data->rsvd = htonl(tcp_seq);
1380         tx_data->flags = htonl(TX_BYPASS_F);
1381         if (tcp_push)
1382                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1383
1384         /* check left again, it might go beyond queue limit */
1385         pos = tx_data + 1;
1386         left = (void *)q->q.stat - pos;
1387
1388         /* check the position again */
1389         if (!left) {
1390                 left = (void *)end - (void *)q->q.stat;
1391                 pos = q->q.desc;
1392                 end = pos + left;
1393         }
1394         /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1395          * bytes of actual IV and 4 bytes of 16 byte-sequence.
1396          */
1397         memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1398         memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1399         *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1400                 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1401                 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1402
1403         pos += 16;
1404         /* Prior_data_len will always be less than 16 bytes, fill the
1405          * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1406          * to 0.
1407          */
1408         if (prior_data_len)
1409                 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1410         /* send the complete packet except the header */
1411         cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1412                         sgl_sdesc->addr);
1413         sgl_sdesc->skb = skb;
1414
1415         chcr_txq_advance(&q->q, ndesc);
1416         cxgb4_ring_tx_db(adap, &q->q, ndesc);
1417
1418         return 0;
1419 }
1420
1421 /*
1422  * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1423  * only plain text (only tls header and iv)
1424  * @tx_info - driver specific tls info.
1425  * @skb - skb contains partial record..
1426  * @tcp_seq
1427  * @mss - segment size.
1428  * @tcp_push - tcp push bit.
1429  * @q - TX queue.
1430  * @port_id : port number
1431  * @perior_data - data before the current segment, required to make this record
1432  *               16 byte aligned.
1433  * @prior_data_len - prior_data length (less than 16)
1434  * return: NETDEV_TX_BUSY/NET_TX_OK.
1435  */
1436 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1437                                  struct sk_buff *skb, u32 tcp_seq, u32 mss,
1438                                  bool tcp_push, struct sge_eth_txq *q,
1439                                  u32 port_id, u8 *prior_data,
1440                                  u32 prior_data_len)
1441 {
1442         int credits, left, len16, last_desc;
1443         unsigned int flits = 0, ndesc;
1444         struct tx_sw_desc *sgl_sdesc;
1445         struct cpl_tx_data *tx_data;
1446         struct ulptx_idata *idata;
1447         struct ulp_txpkt *ulptx;
1448         struct fw_ulptx_wr *wr;
1449         u32 wr_mid = 0;
1450         void *pos;
1451         u64 *end;
1452
1453         flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1454         flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags);
1455         if (prior_data_len)
1456                 flits += 2;
1457         /* WR will need len16 */
1458         len16 = DIV_ROUND_UP(flits, 2);
1459         /* check how many descriptors needed */
1460         ndesc = DIV_ROUND_UP(flits, 8);
1461
1462         credits = chcr_txq_avail(&q->q) - ndesc;
1463         if (unlikely(credits < 0)) {
1464                 chcr_eth_txq_stop(q);
1465                 return NETDEV_TX_BUSY;
1466         }
1467
1468         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1469                 chcr_eth_txq_stop(q);
1470                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1471         }
1472
1473         last_desc = q->q.pidx + ndesc - 1;
1474         if (last_desc >= q->q.size)
1475                 last_desc -= q->q.size;
1476         sgl_sdesc = &q->q.sdesc[last_desc];
1477
1478         if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1479                                    sgl_sdesc->addr) < 0)) {
1480                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1481                 q->mapping_err++;
1482                 return NETDEV_TX_BUSY;
1483         }
1484
1485         pos = &q->q.desc[q->q.pidx];
1486         end = (u64 *)pos + flits;
1487         /* FW_ULPTX_WR */
1488         wr = pos;
1489         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1490         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1491         wr->cookie = 0;
1492         pos += sizeof(*wr);
1493         /* ULP_TXPKT */
1494         ulptx = (struct ulp_txpkt *)(wr + 1);
1495         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1496                         ULP_TXPKT_DATAMODIFY_V(0) |
1497                         ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1498                         ULP_TXPKT_DEST_V(0) |
1499                         ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1500         ulptx->len = htonl(len16 - 1);
1501         /* ULPTX_IDATA sub-command */
1502         idata = (struct ulptx_idata *)(ulptx + 1);
1503         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1504         idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1505         /* CPL_TX_DATA */
1506         tx_data = (struct cpl_tx_data *)(idata + 1);
1507         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1508         tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1509                         TX_LENGTH_V(skb->data_len + prior_data_len));
1510         /* set tcp seq number */
1511         tx_data->rsvd = htonl(tcp_seq);
1512         tx_data->flags = htonl(TX_BYPASS_F);
1513         if (tcp_push)
1514                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1515
1516         pos = tx_data + 1;
1517         /* apart from prior_data_len, we should set remaining part of 16 bytes
1518          * to be zero.
1519          */
1520         if (prior_data_len)
1521                 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1522
1523         /* check left again, it might go beyond queue limit */
1524         left = (void *)q->q.stat - pos;
1525
1526         /* check the position again */
1527         if (!left) {
1528                 left = (void *)end - (void *)q->q.stat;
1529                 pos = q->q.desc;
1530                 end = pos + left;
1531         }
1532         /* send the complete packet including the header */
1533         cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1534                         sgl_sdesc->addr);
1535         sgl_sdesc->skb = skb;
1536
1537         chcr_txq_advance(&q->q, ndesc);
1538         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1539         return 0;
1540 }
1541
1542 /*
1543  * chcr_ktls_copy_record_in_skb
1544  * @nskb - new skb where the frags to be added.
1545  * @record - specific record which has complete 16k record in frags.
1546  */
1547 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1548                                          struct tls_record_info *record)
1549 {
1550         int i = 0;
1551
1552         for (i = 0; i < record->num_frags; i++) {
1553                 skb_shinfo(nskb)->frags[i] = record->frags[i];
1554                 /* increase the frag ref count */
1555                 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1556         }
1557
1558         skb_shinfo(nskb)->nr_frags = record->num_frags;
1559         nskb->data_len = record->len;
1560         nskb->len += record->len;
1561         nskb->truesize += record->len;
1562 }
1563
1564 /*
1565  * chcr_ktls_update_snd_una:  Reset the SEND_UNA. It will be done to avoid
1566  * sending the same segment again. It will discard the segment which is before
1567  * the current tx max.
1568  * @tx_info - driver specific tls info.
1569  * @q - TX queue.
1570  * return: NET_TX_OK/NET_XMIT_DROP.
1571  */
1572 static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info,
1573                                     struct sge_eth_txq *q)
1574 {
1575         struct fw_ulptx_wr *wr;
1576         unsigned int ndesc;
1577         int credits;
1578         void *pos;
1579         u32 len;
1580
1581         len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16);
1582         ndesc = DIV_ROUND_UP(len, 64);
1583
1584         credits = chcr_txq_avail(&q->q) - ndesc;
1585         if (unlikely(credits < 0)) {
1586                 chcr_eth_txq_stop(q);
1587                 return NETDEV_TX_BUSY;
1588         }
1589
1590         pos = &q->q.desc[q->q.pidx];
1591
1592         wr = pos;
1593         /* ULPTX wr */
1594         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1595         wr->cookie = 0;
1596         /* fill len in wr field */
1597         wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
1598
1599         pos += sizeof(*wr);
1600
1601         pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
1602                                          TCB_SND_UNA_RAW_W,
1603                                          TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
1604                                          TCB_SND_UNA_RAW_V(0), 0);
1605
1606         chcr_txq_advance(&q->q, ndesc);
1607         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1608
1609         return 0;
1610 }
1611
1612 /*
1613  * chcr_end_part_handler: This handler will handle the record which
1614  * is complete or if record's end part is received. T6 adapter has a issue that
1615  * it can't send out TAG with partial record so if its an end part then we have
1616  * to send TAG as well and for which we need to fetch the complete record and
1617  * send it to crypto module.
1618  * @tx_info - driver specific tls info.
1619  * @skb - skb contains partial record.
1620  * @record - complete record of 16K size.
1621  * @tcp_seq
1622  * @mss - segment size in which TP needs to chop a packet.
1623  * @tcp_push_no_fin - tcp push if fin is not set.
1624  * @q - TX queue.
1625  * @tls_end_offset - offset from end of the record.
1626  * @last wr : check if this is the last part of the skb going out.
1627  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1628  */
1629 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1630                                  struct sk_buff *skb,
1631                                  struct tls_record_info *record,
1632                                  u32 tcp_seq, int mss, bool tcp_push_no_fin,
1633                                  struct sge_eth_txq *q,
1634                                  u32 tls_end_offset, bool last_wr)
1635 {
1636         struct sk_buff *nskb = NULL;
1637         /* check if it is a complete record */
1638         if (tls_end_offset == record->len) {
1639                 nskb = skb;
1640                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_complete_pkts);
1641         } else {
1642                 dev_kfree_skb_any(skb);
1643
1644                 nskb = alloc_skb(0, GFP_KERNEL);
1645                 if (!nskb)
1646                         return NETDEV_TX_BUSY;
1647                 /* copy complete record in skb */
1648                 chcr_ktls_copy_record_in_skb(nskb, record);
1649                 /* packet is being sent from the beginning, update the tcp_seq
1650                  * accordingly.
1651                  */
1652                 tcp_seq = tls_record_start_seq(record);
1653                 /* reset snd una, so the middle record won't send the already
1654                  * sent part.
1655                  */
1656                 if (chcr_ktls_update_snd_una(tx_info, q))
1657                         goto out;
1658                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_end_pkts);
1659         }
1660
1661         if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1662                                        (last_wr && tcp_push_no_fin),
1663                                        mss)) {
1664                 goto out;
1665         }
1666         return 0;
1667 out:
1668         dev_kfree_skb_any(nskb);
1669         return NETDEV_TX_BUSY;
1670 }
1671
1672 /*
1673  * chcr_short_record_handler: This handler will take care of the records which
1674  * doesn't have end part (1st part or the middle part(/s) of a record). In such
1675  * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1676  * This partial record might be the first part of the record, or the middle
1677  * part. In case of middle record we should fetch the prior data to make it 16
1678  * byte aligned. If it has a partial tls header or iv then get to the start of
1679  * tls header. And if it has partial TAG, then remove the complete TAG and send
1680  * only the payload.
1681  * There is one more possibility that it gets a partial header, send that
1682  * portion as a plaintext.
1683  * @tx_info - driver specific tls info.
1684  * @skb - skb contains partial record..
1685  * @record - complete record of 16K size.
1686  * @tcp_seq
1687  * @mss - segment size in which TP needs to chop a packet.
1688  * @tcp_push_no_fin - tcp push if fin is not set.
1689  * @q - TX queue.
1690  * @tls_end_offset - offset from end of the record.
1691  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1692  */
1693 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1694                                      struct sk_buff *skb,
1695                                      struct tls_record_info *record,
1696                                      u32 tcp_seq, int mss, bool tcp_push_no_fin,
1697                                      struct sge_eth_txq *q, u32 tls_end_offset)
1698 {
1699         u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1700         u8 prior_data[16] = {0};
1701         u32 prior_data_len = 0;
1702         u32 data_len;
1703
1704         /* check if the skb is ending in middle of tag/HASH, its a big
1705          * trouble, send the packet before the HASH.
1706          */
1707         int remaining_record = tls_end_offset - skb->data_len;
1708
1709         if (remaining_record > 0 &&
1710             remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1711                 int trimmed_len = skb->data_len -
1712                         (TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record);
1713                 struct sk_buff *tmp_skb = NULL;
1714                 /* don't process the pkt if it is only a partial tag */
1715                 if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1716                         goto out;
1717
1718                 WARN_ON(trimmed_len > skb->data_len);
1719
1720                 /* shift to those many bytes */
1721                 tmp_skb = alloc_skb(0, GFP_KERNEL);
1722                 if (unlikely(!tmp_skb))
1723                         goto out;
1724
1725                 chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len);
1726                 /* free the last trimmed portion */
1727                 dev_kfree_skb_any(skb);
1728                 skb = tmp_skb;
1729                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_trimmed_pkts);
1730         }
1731         data_len = skb->data_len;
1732         /* check if the middle record's start point is 16 byte aligned. CTR
1733          * needs 16 byte aligned start point to start encryption.
1734          */
1735         if (tls_rec_offset) {
1736                 /* there is an offset from start, means its a middle record */
1737                 int remaining = 0;
1738
1739                 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1740                         prior_data_len = tls_rec_offset;
1741                         tls_rec_offset = 0;
1742                         remaining = 0;
1743                 } else {
1744                         prior_data_len =
1745                                 (tls_rec_offset -
1746                                 (TLS_HEADER_SIZE + tx_info->iv_size))
1747                                 % AES_BLOCK_LEN;
1748                         remaining = tls_rec_offset - prior_data_len;
1749                 }
1750
1751                 /* if prior_data_len is not zero, means we need to fetch prior
1752                  * data to make this record 16 byte aligned, or we need to reach
1753                  * to start offset.
1754                  */
1755                 if (prior_data_len) {
1756                         int i = 0;
1757                         u8 *data = NULL;
1758                         skb_frag_t *f;
1759                         u8 *vaddr;
1760                         int frag_size = 0, frag_delta = 0;
1761
1762                         while (remaining > 0) {
1763                                 frag_size = skb_frag_size(&record->frags[i]);
1764                                 if (remaining < frag_size)
1765                                         break;
1766
1767                                 remaining -= frag_size;
1768                                 i++;
1769                         }
1770                         f = &record->frags[i];
1771                         vaddr = kmap_atomic(skb_frag_page(f));
1772
1773                         data = vaddr + skb_frag_off(f)  + remaining;
1774                         frag_delta = skb_frag_size(f) - remaining;
1775
1776                         if (frag_delta >= prior_data_len) {
1777                                 memcpy(prior_data, data, prior_data_len);
1778                                 kunmap_atomic(vaddr);
1779                         } else {
1780                                 memcpy(prior_data, data, frag_delta);
1781                                 kunmap_atomic(vaddr);
1782                                 /* get the next page */
1783                                 f = &record->frags[i + 1];
1784                                 vaddr = kmap_atomic(skb_frag_page(f));
1785                                 data = vaddr + skb_frag_off(f);
1786                                 memcpy(prior_data + frag_delta,
1787                                        data, (prior_data_len - frag_delta));
1788                                 kunmap_atomic(vaddr);
1789                         }
1790                         /* reset tcp_seq as per the prior_data_required len */
1791                         tcp_seq -= prior_data_len;
1792                         /* include prio_data_len for  further calculation.
1793                          */
1794                         data_len += prior_data_len;
1795                 }
1796                 /* reset snd una, so the middle record won't send the already
1797                  * sent part.
1798                  */
1799                 if (chcr_ktls_update_snd_una(tx_info, q))
1800                         goto out;
1801                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_middle_pkts);
1802         } else {
1803                 /* Else means, its a partial first part of the record. Check if
1804                  * its only the header, don't need to send for encryption then.
1805                  */
1806                 if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) {
1807                         if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1808                                                   tcp_push_no_fin, q,
1809                                                   tx_info->port_id,
1810                                                   prior_data,
1811                                                   prior_data_len)) {
1812                                 goto out;
1813                         }
1814                         return 0;
1815                 }
1816                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_start_pkts);
1817         }
1818
1819         if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1820                                     mss, tls_rec_offset, prior_data,
1821                                     prior_data_len)) {
1822                 goto out;
1823         }
1824
1825         return 0;
1826 out:
1827         dev_kfree_skb_any(skb);
1828         return NETDEV_TX_BUSY;
1829 }
1830
1831 /* nic tls TX handler */
1832 int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1833 {
1834         struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1835         struct tcphdr *th = tcp_hdr(skb);
1836         int data_len, qidx, ret = 0, mss;
1837         struct tls_record_info *record;
1838         struct chcr_stats_debug *stats;
1839         struct chcr_ktls_info *tx_info;
1840         u32 tls_end_offset, tcp_seq;
1841         struct tls_context *tls_ctx;
1842         struct sk_buff *local_skb;
1843         int new_connection_state;
1844         struct sge_eth_txq *q;
1845         struct adapter *adap;
1846         unsigned long flags;
1847
1848         tcp_seq = ntohl(th->seq);
1849
1850         mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len;
1851
1852         /* check if we haven't set it for ktls offload */
1853         if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
1854                 goto out;
1855
1856         tls_ctx = tls_get_ctx(skb->sk);
1857         if (unlikely(tls_ctx->netdev != dev))
1858                 goto out;
1859
1860         tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1861         tx_info = tx_ctx->chcr_info;
1862
1863         if (unlikely(!tx_info))
1864                 goto out;
1865
1866         /* check the connection state, we don't need to pass new connection
1867          * state, state machine will check and update the new state if it is
1868          * stuck due to responses not received from HW.
1869          * Start the tx handling only if state is KTLS_CONN_TX_READY.
1870          */
1871         new_connection_state = chcr_ktls_update_connection_state(tx_info, 0);
1872         if (new_connection_state != KTLS_CONN_TX_READY)
1873                 goto out;
1874
1875         /* don't touch the original skb, make a new skb to extract each records
1876          * and send them separately.
1877          */
1878         local_skb = alloc_skb(0, GFP_KERNEL);
1879
1880         if (unlikely(!local_skb))
1881                 return NETDEV_TX_BUSY;
1882
1883         adap = tx_info->adap;
1884         stats = &adap->chcr_stats;
1885
1886         qidx = skb->queue_mapping;
1887         q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1888         cxgb4_reclaim_completed_tx(adap, &q->q, true);
1889         /* if tcp options are set but finish is not send the options first */
1890         if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1891                 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1892                                                   tx_info->tx_chan);
1893                 if (ret)
1894                         return NETDEV_TX_BUSY;
1895         }
1896         /* update tcb */
1897         ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq),
1898                                       ntohl(th->ack_seq),
1899                                       ntohs(th->window));
1900         if (ret) {
1901                 dev_kfree_skb_any(local_skb);
1902                 return NETDEV_TX_BUSY;
1903         }
1904
1905         /* copy skb contents into local skb */
1906         chcr_ktls_skb_copy(skb, local_skb);
1907
1908         /* go through the skb and send only one record at a time. */
1909         data_len = skb->data_len;
1910         /* TCP segments can be in received either complete or partial.
1911          * chcr_end_part_handler will handle cases if complete record or end
1912          * part of the record is received. Incase of partial end part of record,
1913          * we will send the complete record again.
1914          */
1915
1916         do {
1917                 int i;
1918
1919                 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1920                 /* lock taken */
1921                 spin_lock_irqsave(&tx_ctx->base.lock, flags);
1922                 /* fetch the tls record */
1923                 record = tls_get_record(&tx_ctx->base, tcp_seq,
1924                                         &tx_info->record_no);
1925                 /* By the time packet reached to us, ACK is received, and record
1926                  * won't be found in that case, handle it gracefully.
1927                  */
1928                 if (unlikely(!record)) {
1929                         spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1930                         atomic64_inc(&stats->ktls_tx_drop_no_sync_data);
1931                         goto out;
1932                 }
1933
1934                 if (unlikely(tls_record_is_start_marker(record))) {
1935                         spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1936                         atomic64_inc(&stats->ktls_tx_skip_no_sync_data);
1937                         goto out;
1938                 }
1939
1940                 /* increase page reference count of the record, so that there
1941                  * won't be any chance of page free in middle if in case stack
1942                  * receives ACK and try to delete the record.
1943                  */
1944                 for (i = 0; i < record->num_frags; i++)
1945                         __skb_frag_ref(&record->frags[i]);
1946                 /* lock cleared */
1947                 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1948
1949                 tls_end_offset = record->end_seq - tcp_seq;
1950
1951                 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1952                          tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
1953                 /* if a tls record is finishing in this SKB */
1954                 if (tls_end_offset <= data_len) {
1955                         struct sk_buff *nskb = NULL;
1956
1957                         if (tls_end_offset < data_len) {
1958                                 nskb = alloc_skb(0, GFP_KERNEL);
1959                                 if (unlikely(!nskb)) {
1960                                         ret = -ENOMEM;
1961                                         goto clear_ref;
1962                                 }
1963
1964                                 chcr_ktls_skb_shift(nskb, local_skb,
1965                                                     tls_end_offset);
1966                         } else {
1967                                 /* its the only record in this skb, directly
1968                                  * point it.
1969                                  */
1970                                 nskb = local_skb;
1971                         }
1972                         ret = chcr_end_part_handler(tx_info, nskb, record,
1973                                                     tcp_seq, mss,
1974                                                     (!th->fin && th->psh), q,
1975                                                     tls_end_offset,
1976                                                     (nskb == local_skb));
1977
1978                         if (ret && nskb != local_skb)
1979                                 dev_kfree_skb_any(local_skb);
1980
1981                         data_len -= tls_end_offset;
1982                         /* tcp_seq increment is required to handle next record.
1983                          */
1984                         tcp_seq += tls_end_offset;
1985                 } else {
1986                         ret = chcr_short_record_handler(tx_info, local_skb,
1987                                                         record, tcp_seq, mss,
1988                                                         (!th->fin && th->psh),
1989                                                         q, tls_end_offset);
1990                         data_len = 0;
1991                 }
1992 clear_ref:
1993                 /* clear the frag ref count which increased locally before */
1994                 for (i = 0; i < record->num_frags; i++) {
1995                         /* clear the frag ref count */
1996                         __skb_frag_unref(&record->frags[i]);
1997                 }
1998                 /* if any failure, come out from the loop. */
1999                 if (ret)
2000                         goto out;
2001                 /* length should never be less than 0 */
2002                 WARN_ON(data_len < 0);
2003
2004         } while (data_len > 0);
2005
2006         tx_info->prev_seq = ntohl(th->seq) + skb->data_len;
2007
2008         atomic64_inc(&stats->ktls_tx_encrypted_packets);
2009         atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes);
2010
2011         /* tcp finish is set, send a separate tcp msg including all the options
2012          * as well.
2013          */
2014         if (th->fin)
2015                 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2016
2017 out:
2018         dev_kfree_skb_any(skb);
2019         return NETDEV_TX_OK;
2020 }
2021 #endif /* CONFIG_CHELSIO_TLS_DEVICE */