drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3
   4 #include <net/netfilter/nf_conntrack.h>
   5 #include <net/netfilter/nf_conntrack_core.h>
   6 #include <net/netfilter/nf_conntrack_zones.h>
   7 #include <net/netfilter/nf_conntrack_labels.h>
   8 #include <net/netfilter/nf_conntrack_helper.h>
   9 #include <net/netfilter/nf_conntrack_acct.h>
  10 #include <uapi/linux/tc_act/tc_pedit.h>
  11 #include <net/tc_act/tc_ct.h>
  12 #include <net/flow_offload.h>
  13 #include <net/netfilter/nf_flow_table.h>
  14 #include <linux/workqueue.h>
  15
  16 #include "esw/chains.h"
  17 #include "en/tc_ct.h"
  18 #include "en.h"
  19 #include "en_tc.h"
  20 #include "en_rep.h"
  21
  22 #define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen * 8)
  23 #define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0)
  24 #define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1)
  25 #define MLX5_CT_STATE_TRK_BIT BIT(2)
  26
  27 #define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen * 8)
  28 #define MLX5_FTE_ID_MAX GENMASK(MLX5_FTE_ID_BITS - 1, 0)
  29 #define MLX5_FTE_ID_MASK MLX5_FTE_ID_MAX
  30
  31 #define ct_dbg(fmt, args...)\
  32         netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args)
  33
  34 struct mlx5_tc_ct_priv {
  35         struct mlx5_eswitch *esw;
  36         const struct net_device *netdev;
  37         struct idr fte_ids;
  38         struct idr tuple_ids;
  39         struct rhashtable zone_ht;
  40         struct mlx5_flow_table *ct;
  41         struct mlx5_flow_table *ct_nat;
  42         struct mlx5_flow_table *post_ct;
  43         struct mutex control_lock; /* guards parallel adds/dels */
  44 };
  45
  46 struct mlx5_ct_flow {
  47         struct mlx5_esw_flow_attr pre_ct_attr;
  48         struct mlx5_esw_flow_attr post_ct_attr;
  49         struct mlx5_flow_handle *pre_ct_rule;
  50         struct mlx5_flow_handle *post_ct_rule;
  51         struct mlx5_ct_ft *ft;
  52         u32 fte_id;
  53         u32 chain_mapping;
  54 };
  55
  56 struct mlx5_ct_zone_rule {
  57         struct mlx5_flow_handle *rule;
  58         struct mlx5_esw_flow_attr attr;
  59         int tupleid;
  60         bool nat;
  61 };
  62
  63 struct mlx5_ct_ft {
  64         struct rhash_head node;
  65         u16 zone;
  66         refcount_t refcount;
  67         struct nf_flowtable *nf_ft;
  68         struct mlx5_tc_ct_priv *ct_priv;
  69         struct rhashtable ct_entries_ht;
  70         struct list_head ct_entries_list;
  71 };
  72
  73 struct mlx5_ct_entry {
  74         struct list_head list;
  75         u16 zone;
  76         struct rhash_head node;
  77         struct flow_rule *flow_rule;
  78         struct mlx5_fc *counter;
  79         unsigned long lastuse;
  80         unsigned long cookie;
  81         unsigned long restore_cookie;
  82         struct mlx5_ct_zone_rule zone_rules[2];
  83 };
  84
  85 static const struct rhashtable_params cts_ht_params = {
  86         .head_offset = offsetof(struct mlx5_ct_entry, node),
  87         .key_offset = offsetof(struct mlx5_ct_entry, cookie),
  88         .key_len = sizeof(((struct mlx5_ct_entry *)0)->cookie),
  89         .automatic_shrinking = true,
  90         .min_size = 16 * 1024,
  91 };
  92
  93 static const struct rhashtable_params zone_params = {
  94         .head_offset = offsetof(struct mlx5_ct_ft, node),
  95         .key_offset = offsetof(struct mlx5_ct_ft, zone),
  96         .key_len = sizeof(((struct mlx5_ct_ft *)0)->zone),
  97         .automatic_shrinking = true,
  98 };
  99
 100 static struct mlx5_tc_ct_priv *
 101 mlx5_tc_ct_get_ct_priv(struct mlx5e_priv *priv)
 102 {
 103         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 104         struct mlx5_rep_uplink_priv *uplink_priv;
 105         struct mlx5e_rep_priv *uplink_rpriv;
 106
 107         uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 108         uplink_priv = &uplink_rpriv->uplink_priv;
 109         return uplink_priv->ct_priv;
 110 }
 111
 112 static int
 113 mlx5_tc_ct_set_tuple_match(struct mlx5_flow_spec *spec,
 114                            struct flow_rule *rule)
 115 {
 116         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 117                                        outer_headers);
 118         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 119                                        outer_headers);
 120         u16 addr_type = 0;
 121         u8 ip_proto = 0;
 122
 123         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
 124                 struct flow_match_basic match;
 125
 126                 flow_rule_match_basic(rule, &match);
 127
 128                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
 129                          ntohs(match.mask->n_proto));
 130                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
 131                          ntohs(match.key->n_proto));
 132                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 133                          match.mask->ip_proto);
 134                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 135                          match.key->ip_proto);
 136
 137                 ip_proto = match.key->ip_proto;
 138         }
 139
 140         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
 141                 struct flow_match_control match;
 142
 143                 flow_rule_match_control(rule, &match);
 144                 addr_type = match.key->addr_type;
 145         }
 146
 147         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 148                 struct flow_match_ipv4_addrs match;
 149
 150                 flow_rule_match_ipv4_addrs(rule, &match);
 151                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 152                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
 153                        &match.mask->src, sizeof(match.mask->src));
 154                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 155                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
 156                        &match.key->src, sizeof(match.key->src));
 157                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 158                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 159                        &match.mask->dst, sizeof(match.mask->dst));
 160                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 161                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 162                        &match.key->dst, sizeof(match.key->dst));
 163         }
 164
 165         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 166                 struct flow_match_ipv6_addrs match;
 167
 168                 flow_rule_match_ipv6_addrs(rule, &match);
 169                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 170                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 171                        &match.mask->src, sizeof(match.mask->src));
 172                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 173                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 174                        &match.key->src, sizeof(match.key->src));
 175
 176                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 177                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 178                        &match.mask->dst, sizeof(match.mask->dst));
 179                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 180                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 181                        &match.key->dst, sizeof(match.key->dst));
 182         }
 183
 184         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 185                 struct flow_match_ports match;
 186
 187                 flow_rule_match_ports(rule, &match);
 188                 switch (ip_proto) {
 189                 case IPPROTO_TCP:
 190                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 191                                  tcp_sport, ntohs(match.mask->src));
 192                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 193                                  tcp_sport, ntohs(match.key->src));
 194
 195                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 196                                  tcp_dport, ntohs(match.mask->dst));
 197                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 198                                  tcp_dport, ntohs(match.key->dst));
 199                         break;
 200
 201                 case IPPROTO_UDP:
 202                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 203                                  udp_sport, ntohs(match.mask->src));
 204                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 205                                  udp_sport, ntohs(match.key->src));
 206
 207                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 208                                  udp_dport, ntohs(match.mask->dst));
 209                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 210                                  udp_dport, ntohs(match.key->dst));
 211                         break;
 212                 default:
 213                         break;
 214                 }
 215         }
 216
 217         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
 218                 struct flow_match_tcp match;
 219
 220                 flow_rule_match_tcp(rule, &match);
 221                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
 222                          ntohs(match.mask->flags));
 223                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
 224                          ntohs(match.key->flags));
 225         }
 226
 227         return 0;
 228 }
 229
 230 static void
 231 mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv,
 232                           struct mlx5_ct_entry *entry,
 233                           bool nat)
 234 {
 235         struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
 236         struct mlx5_esw_flow_attr *attr = &zone_rule->attr;
 237         struct mlx5_eswitch *esw = ct_priv->esw;
 238
 239         ct_dbg("Deleting ct entry rule in zone %d", entry->zone);
 240
 241         mlx5_eswitch_del_offloaded_rule(esw, zone_rule->rule, attr);
 242         mlx5_modify_header_dealloc(esw->dev, attr->modify_hdr);
 243         idr_remove(&ct_priv->tuple_ids, zone_rule->tupleid);
 244 }
 245
 246 static void
 247 mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv,
 248                            struct mlx5_ct_entry *entry)
 249 {
 250         mlx5_tc_ct_entry_del_rule(ct_priv, entry, true);
 251         mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
 252
 253         mlx5_fc_destroy(ct_priv->esw->dev, entry->counter);
 254 }
 255
 256 static struct flow_action_entry *
 257 mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule)
 258 {
 259         struct flow_action *flow_action = &flow_rule->action;
 260         struct flow_action_entry *act;
 261         int i;
 262
 263         flow_action_for_each(i, act, flow_action) {
 264                 if (act->id == FLOW_ACTION_CT_METADATA)
 265                         return act;
 266         }
 267
 268         return NULL;
 269 }
 270
 271 static int
 272 mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv,
 273                                struct mlx5e_tc_mod_hdr_acts *mod_acts,
 274                                u8 ct_state,
 275                                u32 mark,
 276                                u32 label,
 277                                u32 tupleid)
 278 {
 279         struct mlx5_eswitch *esw = ct_priv->esw;
 280         int err;
 281
 282         err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
 283                                         CTSTATE_TO_REG, ct_state);
 284         if (err)
 285                 return err;
 286
 287         err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
 288                                         MARK_TO_REG, mark);
 289         if (err)
 290                 return err;
 291
 292         err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
 293                                         LABELS_TO_REG, label);
 294         if (err)
 295                 return err;
 296
 297         err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
 298                                         TUPLEID_TO_REG, tupleid);
 299         if (err)
 300                 return err;
 301
 302         return 0;
 303 }
 304
 305 static int
 306 mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
 307                                    char *modact)
 308 {
 309         u32 offset = act->mangle.offset, field;
 310
 311         switch (act->mangle.htype) {
 312         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 313                 MLX5_SET(set_action_in, modact, length, 0);
 314                 if (offset == offsetof(struct iphdr, saddr))
 315                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV4;
 316                 else if (offset == offsetof(struct iphdr, daddr))
 317                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV4;
 318                 else
 319                         return -EOPNOTSUPP;
 320                 break;
 321
 322         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 323                 MLX5_SET(set_action_in, modact, length, 0);
 324                 if (offset == offsetof(struct ipv6hdr, saddr))
 325                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0;
 326                 else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
 327                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
 328                 else if (offset == offsetof(struct ipv6hdr, saddr) + 8)
 329                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64;
 330                 else if (offset == offsetof(struct ipv6hdr, saddr) + 12)
 331                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96;
 332                 else if (offset == offsetof(struct ipv6hdr, daddr))
 333                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0;
 334                 else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
 335                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
 336                 else if (offset == offsetof(struct ipv6hdr, daddr) + 8)
 337                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64;
 338                 else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
 339                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96;
 340                 else
 341                         return -EOPNOTSUPP;
 342                 break;
 343
 344         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 345                 MLX5_SET(set_action_in, modact, length, 16);
 346                 if (offset == offsetof(struct tcphdr, source))
 347                         field = MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT;
 348                 else if (offset == offsetof(struct tcphdr, dest))
 349                         field = MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT;
 350                 else
 351                         return -EOPNOTSUPP;
 352                 break;
 353
 354         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 355                 MLX5_SET(set_action_in, modact, length, 16);
 356                 if (offset == offsetof(struct udphdr, source))
 357                         field = MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT;
 358                 else if (offset == offsetof(struct udphdr, dest))
 359                         field = MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT;
 360                 else
 361                         return -EOPNOTSUPP;
 362                 break;
 363
 364         default:
 365                 return -EOPNOTSUPP;
 366         }
 367
 368         MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
 369         MLX5_SET(set_action_in, modact, offset, 0);
 370         MLX5_SET(set_action_in, modact, field, field);
 371         MLX5_SET(set_action_in, modact, data, act->mangle.val);
 372
 373         return 0;
 374 }
 375
 376 static int
 377 mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv,
 378                             struct flow_rule *flow_rule,
 379                             struct mlx5e_tc_mod_hdr_acts *mod_acts)
 380 {
 381         struct flow_action *flow_action = &flow_rule->action;
 382         struct mlx5_core_dev *mdev = ct_priv->esw->dev;
 383         struct flow_action_entry *act;
 384         size_t action_size;
 385         char *modact;
 386         int err, i;
 387
 388         action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto);
 389
 390         flow_action_for_each(i, act, flow_action) {
 391                 switch (act->id) {
 392                 case FLOW_ACTION_MANGLE: {
 393                         err = alloc_mod_hdr_actions(mdev,
 394                                                     MLX5_FLOW_NAMESPACE_FDB,
 395                                                     mod_acts);
 396                         if (err)
 397                                 return err;
 398
 399                         modact = mod_acts->actions +
 400                                  mod_acts->num_actions * action_size;
 401
 402                         err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact);
 403                         if (err)
 404                                 return err;
 405
 406                         mod_acts->num_actions++;
 407                 }
 408                 break;
 409
 410                 case FLOW_ACTION_CT_METADATA:
 411                         /* Handled earlier */
 412                         continue;
 413                 default:
 414                         return -EOPNOTSUPP;
 415                 }
 416         }
 417
 418         return 0;
 419 }
 420
 421 static int
 422 mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 423                                 struct mlx5_esw_flow_attr *attr,
 424                                 struct flow_rule *flow_rule,
 425                                 u32 tupleid,
 426                                 bool nat)
 427 {
 428         struct mlx5e_tc_mod_hdr_acts mod_acts = {};
 429         struct mlx5_eswitch *esw = ct_priv->esw;
 430         struct mlx5_modify_hdr *mod_hdr;
 431         struct flow_action_entry *meta;
 432         int err;
 433
 434         meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
 435         if (!meta)
 436                 return -EOPNOTSUPP;
 437
 438         if (meta->ct_metadata.labels[1] ||
 439             meta->ct_metadata.labels[2] ||
 440             meta->ct_metadata.labels[3]) {
 441                 ct_dbg("Failed to offload ct entry due to unsupported label");
 442                 return -EOPNOTSUPP;
 443         }
 444
 445         if (nat) {
 446                 err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule,
 447                                                   &mod_acts);
 448                 if (err)
 449                         goto err_mapping;
 450         }
 451
 452         err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,
 453                                              (MLX5_CT_STATE_ESTABLISHED_BIT |
 454                                               MLX5_CT_STATE_TRK_BIT),
 455                                              meta->ct_metadata.mark,
 456                                              meta->ct_metadata.labels[0],
 457                                              tupleid);
 458         if (err)
 459                 goto err_mapping;
 460
 461         mod_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB,
 462                                            mod_acts.num_actions,
 463                                            mod_acts.actions);
 464         if (IS_ERR(mod_hdr)) {
 465                 err = PTR_ERR(mod_hdr);
 466                 goto err_mapping;
 467         }
 468         attr->modify_hdr = mod_hdr;
 469
 470         dealloc_mod_hdr_actions(&mod_acts);
 471         return 0;
 472
 473 err_mapping:
 474         dealloc_mod_hdr_actions(&mod_acts);
 475         return err;
 476 }
 477
 478 static int
 479 mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
 480                           struct flow_rule *flow_rule,
 481                           struct mlx5_ct_entry *entry,
 482                           bool nat)
 483 {
 484         struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
 485         struct mlx5_esw_flow_attr *attr = &zone_rule->attr;
 486         struct mlx5_eswitch *esw = ct_priv->esw;
 487         struct mlx5_flow_spec *spec = NULL;
 488         u32 tupleid = 1;
 489         int err;
 490
 491         zone_rule->nat = nat;
 492
 493         spec = kzalloc(sizeof(*spec), GFP_KERNEL);
 494         if (!spec)
 495                 return -ENOMEM;
 496
 497         /* Get tuple unique id */
 498         err = idr_alloc_u32(&ct_priv->tuple_ids, zone_rule, &tupleid,
 499                             TUPLE_ID_MAX, GFP_KERNEL);
 500         if (err) {
 501                 netdev_warn(ct_priv->netdev,
 502                             "Failed to allocate tuple id, err: %d\n", err);
 503                 goto err_idr_alloc;
 504         }
 505         zone_rule->tupleid = tupleid;
 506
 507         err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule,
 508                                               tupleid, nat);
 509         if (err) {
 510                 ct_dbg("Failed to create ct entry mod hdr");
 511                 goto err_mod_hdr;
 512         }
 513
 514         attr->action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
 515                        MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 516                        MLX5_FLOW_CONTEXT_ACTION_COUNT;
 517         attr->dest_chain = 0;
 518         attr->dest_ft = ct_priv->post_ct;
 519         attr->fdb = nat ? ct_priv->ct_nat : ct_priv->ct;
 520         attr->outer_match_level = MLX5_MATCH_L4;
 521         attr->counter = entry->counter;
 522         attr->flags |= MLX5_ESW_ATTR_FLAG_NO_IN_PORT;
 523
 524         mlx5_tc_ct_set_tuple_match(spec, flow_rule);
 525         mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
 526                                     entry->zone & MLX5_CT_ZONE_MASK,
 527                                     MLX5_CT_ZONE_MASK);
 528
 529         zone_rule->rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
 530         if (IS_ERR(zone_rule->rule)) {
 531                 err = PTR_ERR(zone_rule->rule);
 532                 ct_dbg("Failed to add ct entry rule, nat: %d", nat);
 533                 goto err_rule;
 534         }
 535
 536         kfree(spec);
 537         ct_dbg("Offloaded ct entry rule in zone %d", entry->zone);
 538
 539         return 0;
 540
 541 err_rule:
 542         mlx5_modify_header_dealloc(esw->dev, attr->modify_hdr);
 543 err_mod_hdr:
 544         idr_remove(&ct_priv->tuple_ids, zone_rule->tupleid);
 545 err_idr_alloc:
 546         kfree(spec);
 547         return err;
 548 }
 549
 550 static int
 551 mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv,
 552                            struct flow_rule *flow_rule,
 553                            struct mlx5_ct_entry *entry)
 554 {
 555         struct mlx5_eswitch *esw = ct_priv->esw;
 556         int err;
 557
 558         entry->counter = mlx5_fc_create(esw->dev, true);
 559         if (IS_ERR(entry->counter)) {
 560                 err = PTR_ERR(entry->counter);
 561                 ct_dbg("Failed to create counter for ct entry");
 562                 return err;
 563         }
 564
 565         err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false);
 566         if (err)
 567                 goto err_orig;
 568
 569         err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true);
 570         if (err)
 571                 goto err_nat;
 572
 573         return 0;
 574
 575 err_nat:
 576         mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
 577 err_orig:
 578         mlx5_fc_destroy(esw->dev, entry->counter);
 579         return err;
 580 }
 581
 582 static int
 583 mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
 584                                   struct flow_cls_offload *flow)
 585 {
 586         struct flow_rule *flow_rule = flow_cls_offload_flow_rule(flow);
 587         struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
 588         struct flow_action_entry *meta_action;
 589         unsigned long cookie = flow->cookie;
 590         struct mlx5_ct_entry *entry;
 591         int err;
 592
 593         meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
 594         if (!meta_action)
 595                 return -EOPNOTSUPP;
 596
 597         entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie,
 598                                        cts_ht_params);
 599         if (entry)
 600                 return 0;
 601
 602         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 603         if (!entry)
 604                 return -ENOMEM;
 605
 606         entry->zone = ft->zone;
 607         entry->flow_rule = flow_rule;
 608         entry->cookie = flow->cookie;
 609         entry->restore_cookie = meta_action->ct_metadata.cookie;
 610
 611         err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry);
 612         if (err)
 613                 goto err_rules;
 614
 615         err = rhashtable_insert_fast(&ft->ct_entries_ht, &entry->node,
 616                                      cts_ht_params);
 617         if (err)
 618                 goto err_insert;
 619
 620         list_add(&entry->list, &ft->ct_entries_list);
 621
 622         return 0;
 623
 624 err_insert:
 625         mlx5_tc_ct_entry_del_rules(ct_priv, entry);
 626 err_rules:
 627         kfree(entry);
 628         netdev_warn(ct_priv->netdev,
 629                     "Failed to offload ct entry, err: %d\n", err);
 630         return err;
 631 }
 632
 633 static int
 634 mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft,
 635                                   struct flow_cls_offload *flow)
 636 {
 637         unsigned long cookie = flow->cookie;
 638         struct mlx5_ct_entry *entry;
 639
 640         entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie,
 641                                        cts_ht_params);
 642         if (!entry)
 643                 return -ENOENT;
 644
 645         mlx5_tc_ct_entry_del_rules(ft->ct_priv, entry);
 646         WARN_ON(rhashtable_remove_fast(&ft->ct_entries_ht,
 647                                        &entry->node,
 648                                        cts_ht_params));
 649         list_del(&entry->list);
 650         kfree(entry);
 651
 652         return 0;
 653 }
 654
 655 static int
 656 mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft,
 657                                     struct flow_cls_offload *f)
 658 {
 659         unsigned long cookie = f->cookie;
 660         struct mlx5_ct_entry *entry;
 661         u64 lastuse, packets, bytes;
 662
 663         entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie,
 664                                        cts_ht_params);
 665         if (!entry)
 666                 return -ENOENT;
 667
 668         mlx5_fc_query_cached(entry->counter, &bytes, &packets, &lastuse);
 669         flow_stats_update(&f->stats, bytes, packets, lastuse,
 670                           FLOW_ACTION_HW_STATS_DELAYED);
 671
 672         return 0;
 673 }
 674
 675 static int
 676 mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data,
 677                               void *cb_priv)
 678 {
 679         struct flow_cls_offload *f = type_data;
 680         struct mlx5_ct_ft *ft = cb_priv;
 681
 682         if (type != TC_SETUP_CLSFLOWER)
 683                 return -EOPNOTSUPP;
 684
 685         switch (f->command) {
 686         case FLOW_CLS_REPLACE:
 687                 return mlx5_tc_ct_block_flow_offload_add(ft, f);
 688         case FLOW_CLS_DESTROY:
 689                 return mlx5_tc_ct_block_flow_offload_del(ft, f);
 690         case FLOW_CLS_STATS:
 691                 return mlx5_tc_ct_block_flow_offload_stats(ft, f);
 692         default:
 693                 break;
 694         };
 695
 696         return -EOPNOTSUPP;
 697 }
 698
 699 int
 700 mlx5_tc_ct_parse_match(struct mlx5e_priv *priv,
 701                        struct mlx5_flow_spec *spec,
 702                        struct flow_cls_offload *f,
 703                        struct netlink_ext_ack *extack)
 704 {
 705         struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
 706         struct flow_dissector_key_ct *mask, *key;
 707         bool trk, est, untrk, unest, new;
 708         u32 ctstate = 0, ctstate_mask = 0;
 709         u16 ct_state_on, ct_state_off;
 710         u16 ct_state, ct_state_mask;
 711         struct flow_match_ct match;
 712
 713         if (!flow_rule_match_key(f->rule, FLOW_DISSECTOR_KEY_CT))
 714                 return 0;
 715
 716         if (!ct_priv) {
 717                 NL_SET_ERR_MSG_MOD(extack,
 718                                    "offload of ct matching isn't available");
 719                 return -EOPNOTSUPP;
 720         }
 721
 722         flow_rule_match_ct(f->rule, &match);
 723
 724         key = match.key;
 725         mask = match.mask;
 726
 727         ct_state = key->ct_state;
 728         ct_state_mask = mask->ct_state;
 729
 730         if (ct_state_mask & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
 731                               TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED |
 732                               TCA_FLOWER_KEY_CT_FLAGS_NEW)) {
 733                 NL_SET_ERR_MSG_MOD(extack,
 734                                    "only ct_state trk, est and new are supported for offload");
 735                 return -EOPNOTSUPP;
 736         }
 737
 738         if (mask->ct_labels[1] || mask->ct_labels[2] || mask->ct_labels[3]) {
 739                 NL_SET_ERR_MSG_MOD(extack,
 740                                    "only lower 32bits of ct_labels are supported for offload");
 741                 return -EOPNOTSUPP;
 742         }
 743
 744         ct_state_on = ct_state & ct_state_mask;
 745         ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask;
 746         trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
 747         new = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_NEW;
 748         est = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
 749         untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
 750         unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
 751
 752         ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;
 753         ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
 754         ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;
 755         ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
 756
 757         if (new) {
 758                 NL_SET_ERR_MSG_MOD(extack,
 759                                    "matching on ct_state +new isn't supported");
 760                 return -EOPNOTSUPP;
 761         }
 762
 763         if (mask->ct_zone)
 764                 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
 765                                             key->ct_zone, MLX5_CT_ZONE_MASK);
 766         if (ctstate_mask)
 767                 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
 768                                             ctstate, ctstate_mask);
 769         if (mask->ct_mark)
 770                 mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG,
 771                                             key->ct_mark, mask->ct_mark);
 772         if (mask->ct_labels[0])
 773                 mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG,
 774                                             key->ct_labels[0],
 775                                             mask->ct_labels[0]);
 776
 777         return 0;
 778 }
 779
 780 int
 781 mlx5_tc_ct_parse_action(struct mlx5e_priv *priv,
 782                         struct mlx5_esw_flow_attr *attr,
 783                         const struct flow_action_entry *act,
 784                         struct netlink_ext_ack *extack)
 785 {
 786         struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
 787
 788         if (!ct_priv) {
 789                 NL_SET_ERR_MSG_MOD(extack,
 790                                    "offload of ct action isn't available");
 791                 return -EOPNOTSUPP;
 792         }
 793
 794         attr->ct_attr.zone = act->ct.zone;
 795         attr->ct_attr.ct_action = act->ct.action;
 796         attr->ct_attr.nf_ft = act->ct.flow_table;
 797
 798         return 0;
 799 }
 800
 801 static struct mlx5_ct_ft *
 802 mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone,
 803                      struct nf_flowtable *nf_ft)
 804 {
 805         struct mlx5_ct_ft *ft;
 806         int err;
 807
 808         ft = rhashtable_lookup_fast(&ct_priv->zone_ht, &zone, zone_params);
 809         if (ft) {
 810                 refcount_inc(&ft->refcount);
 811                 return ft;
 812         }
 813
 814         ft = kzalloc(sizeof(*ft), GFP_KERNEL);
 815         if (!ft)
 816                 return ERR_PTR(-ENOMEM);
 817
 818         ft->zone = zone;
 819         ft->nf_ft = nf_ft;
 820         ft->ct_priv = ct_priv;
 821         INIT_LIST_HEAD(&ft->ct_entries_list);
 822         refcount_set(&ft->refcount, 1);
 823
 824         err = rhashtable_init(&ft->ct_entries_ht, &cts_ht_params);
 825         if (err)
 826                 goto err_init;
 827
 828         err = rhashtable_insert_fast(&ct_priv->zone_ht, &ft->node,
 829                                      zone_params);
 830         if (err)
 831                 goto err_insert;
 832
 833         err = nf_flow_table_offload_add_cb(ft->nf_ft,
 834                                            mlx5_tc_ct_block_flow_offload, ft);
 835         if (err)
 836                 goto err_add_cb;
 837
 838         return ft;
 839
 840 err_add_cb:
 841         rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
 842 err_insert:
 843         rhashtable_destroy(&ft->ct_entries_ht);
 844 err_init:
 845         kfree(ft);
 846         return ERR_PTR(err);
 847 }
 848
 849 static void
 850 mlx5_tc_ct_flush_ft(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
 851 {
 852         struct mlx5_ct_entry *entry;
 853
 854         list_for_each_entry(entry, &ft->ct_entries_list, list)
 855                 mlx5_tc_ct_entry_del_rules(ft->ct_priv, entry);
 856 }
 857
 858 static void
 859 mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
 860 {
 861         if (!refcount_dec_and_test(&ft->refcount))
 862                 return;
 863
 864         nf_flow_table_offload_del_cb(ft->nf_ft,
 865                                      mlx5_tc_ct_block_flow_offload, ft);
 866         mlx5_tc_ct_flush_ft(ct_priv, ft);
 867         rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
 868         rhashtable_destroy(&ft->ct_entries_ht);
 869         kfree(ft);
 870 }
 871
 872 /* We translate the tc filter with CT action to the following HW model:
 873  *
 874  * +-------------------+      +--------------------+    +--------------+
 875  * + pre_ct (tc chain) +----->+ CT (nat or no nat) +--->+ post_ct      +----->
 876  * + original match    +  |   + tuple + zone match + |  + fte_id match +  |
 877  * +-------------------+  |   +--------------------+ |  +--------------+  |
 878  *                        v                          v                    v
 879  *                       set chain miss mapping  set mark             original
 880  *                       set fte_id              set label            filter
 881  *                       set zone                set established      actions
 882  *                       set tunnel_id           do nat (if needed)
 883  *                       do decap
 884  */
 885 static int
 886 __mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv,
 887                           struct mlx5e_tc_flow *flow,
 888                           struct mlx5_flow_spec *orig_spec,
 889                           struct mlx5_esw_flow_attr *attr,
 890                           struct mlx5_flow_handle **flow_rule)
 891 {
 892         struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
 893         bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
 894         struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {};
 895         struct mlx5_flow_spec *post_ct_spec = NULL;
 896         struct mlx5_eswitch *esw = ct_priv->esw;
 897         struct mlx5_esw_flow_attr *pre_ct_attr;
 898         struct  mlx5_modify_hdr *mod_hdr;
 899         struct mlx5_flow_handle *rule;
 900         struct mlx5_ct_flow *ct_flow;
 901         int chain_mapping = 0, err;
 902         struct mlx5_ct_ft *ft;
 903         u32 fte_id = 1;
 904
 905         post_ct_spec = kzalloc(sizeof(*post_ct_spec), GFP_KERNEL);
 906         ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
 907         if (!post_ct_spec || !ct_flow) {
 908                 kfree(post_ct_spec);
 909                 kfree(ct_flow);
 910                 return -ENOMEM;
 911         }
 912
 913         /* Register for CT established events */
 914         ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone,
 915                                   attr->ct_attr.nf_ft);
 916         if (IS_ERR(ft)) {
 917                 err = PTR_ERR(ft);
 918                 ct_dbg("Failed to register to ft callback");
 919                 goto err_ft;
 920         }
 921         ct_flow->ft = ft;
 922
 923         err = idr_alloc_u32(&ct_priv->fte_ids, ct_flow, &fte_id,
 924                             MLX5_FTE_ID_MAX, GFP_KERNEL);
 925         if (err) {
 926                 netdev_warn(priv->netdev,
 927                             "Failed to allocate fte id, err: %d\n", err);
 928                 goto err_idr;
 929         }
 930         ct_flow->fte_id = fte_id;
 931
 932         /* Base esw attributes of both rules on original rule attribute */
 933         pre_ct_attr = &ct_flow->pre_ct_attr;
 934         memcpy(pre_ct_attr, attr, sizeof(*attr));
 935         memcpy(&ct_flow->post_ct_attr, attr, sizeof(*attr));
 936
 937         /* Modify the original rule's action to fwd and modify, leave decap */
 938         pre_ct_attr->action = attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP;
 939         pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 940                                MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
 941
 942         /* Write chain miss tag for miss in ct table as we
 943          * don't go though all prios of this chain as normal tc rules
 944          * miss.
 945          */
 946         err = mlx5_esw_chains_get_chain_mapping(esw, attr->chain,
 947                                                 &chain_mapping);
 948         if (err) {
 949                 ct_dbg("Failed to get chain register mapping for chain");
 950                 goto err_get_chain;
 951         }
 952         ct_flow->chain_mapping = chain_mapping;
 953
 954         err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts,
 955                                         CHAIN_TO_REG, chain_mapping);
 956         if (err) {
 957                 ct_dbg("Failed to set chain register mapping");
 958                 goto err_mapping;
 959         }
 960
 961         err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts, ZONE_TO_REG,
 962                                         attr->ct_attr.zone &
 963                                         MLX5_CT_ZONE_MASK);
 964         if (err) {
 965                 ct_dbg("Failed to set zone register mapping");
 966                 goto err_mapping;
 967         }
 968
 969         err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts,
 970                                         FTEID_TO_REG, fte_id);
 971         if (err) {
 972                 ct_dbg("Failed to set fte_id register mapping");
 973                 goto err_mapping;
 974         }
 975
 976         /* If original flow is decap, we do it before going into ct table
 977          * so add a rewrite for the tunnel match_id.
 978          */
 979         if ((pre_ct_attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) &&
 980             attr->chain == 0) {
 981                 u32 tun_id = mlx5e_tc_get_flow_tun_id(flow);
 982
 983                 err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts,
 984                                                 TUNNEL_TO_REG,
 985                                                 tun_id);
 986                 if (err) {
 987                         ct_dbg("Failed to set tunnel register mapping");
 988                         goto err_mapping;
 989                 }
 990         }
 991
 992         mod_hdr = mlx5_modify_header_alloc(esw->dev,
 993                                            MLX5_FLOW_NAMESPACE_FDB,
 994                                            pre_mod_acts.num_actions,
 995                                            pre_mod_acts.actions);
 996         if (IS_ERR(mod_hdr)) {
 997                 err = PTR_ERR(mod_hdr);
 998                 ct_dbg("Failed to create pre ct mod hdr");
 999                 goto err_mapping;
1000         }
1001         pre_ct_attr->modify_hdr = mod_hdr;
1002
1003         /* Post ct rule matches on fte_id and executes original rule's
1004          * tc rule action
1005          */
1006         mlx5e_tc_match_to_reg_match(post_ct_spec, FTEID_TO_REG,
1007                                     fte_id, MLX5_FTE_ID_MASK);
1008
1009         /* Put post_ct rule on post_ct fdb */
1010         ct_flow->post_ct_attr.chain = 0;
1011         ct_flow->post_ct_attr.prio = 0;
1012         ct_flow->post_ct_attr.fdb = ct_priv->post_ct;
1013
1014         ct_flow->post_ct_attr.inner_match_level = MLX5_MATCH_NONE;
1015         ct_flow->post_ct_attr.outer_match_level = MLX5_MATCH_NONE;
1016         ct_flow->post_ct_attr.action &= ~(MLX5_FLOW_CONTEXT_ACTION_DECAP);
1017         rule = mlx5_eswitch_add_offloaded_rule(esw, post_ct_spec,
1018                                                &ct_flow->post_ct_attr);
1019         ct_flow->post_ct_rule = rule;
1020         if (IS_ERR(ct_flow->post_ct_rule)) {
1021                 err = PTR_ERR(ct_flow->post_ct_rule);
1022                 ct_dbg("Failed to add post ct rule");
1023                 goto err_insert_post_ct;
1024         }
1025
1026         /* Change original rule point to ct table */
1027         pre_ct_attr->dest_chain = 0;
1028         pre_ct_attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct;
1029         ct_flow->pre_ct_rule = mlx5_eswitch_add_offloaded_rule(esw,
1030                                                                orig_spec,
1031                                                                pre_ct_attr);
1032         if (IS_ERR(ct_flow->pre_ct_rule)) {
1033                 err = PTR_ERR(ct_flow->pre_ct_rule);
1034                 ct_dbg("Failed to add pre ct rule");
1035                 goto err_insert_orig;
1036         }
1037
1038         attr->ct_attr.ct_flow = ct_flow;
1039         *flow_rule = ct_flow->post_ct_rule;
1040         dealloc_mod_hdr_actions(&pre_mod_acts);
1041         kfree(post_ct_spec);
1042
1043         return 0;
1044
1045 err_insert_orig:
1046         mlx5_eswitch_del_offloaded_rule(ct_priv->esw, ct_flow->post_ct_rule,
1047                                         &ct_flow->post_ct_attr);
1048 err_insert_post_ct:
1049         mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
1050 err_mapping:
1051         dealloc_mod_hdr_actions(&pre_mod_acts);
1052         mlx5_esw_chains_put_chain_mapping(esw, ct_flow->chain_mapping);
1053 err_get_chain:
1054         idr_remove(&ct_priv->fte_ids, fte_id);
1055 err_idr:
1056         mlx5_tc_ct_del_ft_cb(ct_priv, ft);
1057 err_ft:
1058         kfree(post_ct_spec);
1059         kfree(ct_flow);
1060         netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err);
1061         return err;
1062 }
1063
1064 static int
1065 __mlx5_tc_ct_flow_offload_clear(struct mlx5e_priv *priv,
1066                                 struct mlx5e_tc_flow *flow,
1067                                 struct mlx5_flow_spec *orig_spec,
1068                                 struct mlx5_esw_flow_attr *attr,
1069                                 struct mlx5e_tc_mod_hdr_acts *mod_acts,
1070                                 struct mlx5_flow_handle **flow_rule)
1071 {
1072         struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
1073         struct mlx5_eswitch *esw = ct_priv->esw;
1074         struct mlx5_esw_flow_attr *pre_ct_attr;
1075         struct mlx5_modify_hdr *mod_hdr;
1076         struct mlx5_flow_handle *rule;
1077         struct mlx5_ct_flow *ct_flow;
1078         int err;
1079
1080         ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
1081         if (!ct_flow)
1082                 return -ENOMEM;
1083
1084         /* Base esw attributes on original rule attribute */
1085         pre_ct_attr = &ct_flow->pre_ct_attr;
1086         memcpy(pre_ct_attr, attr, sizeof(*attr));
1087
1088         err = mlx5_tc_ct_entry_set_registers(ct_priv, mod_acts, 0, 0, 0, 0);
1089         if (err) {
1090                 ct_dbg("Failed to set register for ct clear");
1091                 goto err_set_registers;
1092         }
1093
1094         mod_hdr = mlx5_modify_header_alloc(esw->dev,
1095                                            MLX5_FLOW_NAMESPACE_FDB,
1096                                            mod_acts->num_actions,
1097                                            mod_acts->actions);
1098         if (IS_ERR(mod_hdr)) {
1099                 err = PTR_ERR(mod_hdr);
1100                 ct_dbg("Failed to add create ct clear mod hdr");
1101                 goto err_set_registers;
1102         }
1103
1104         dealloc_mod_hdr_actions(mod_acts);
1105         pre_ct_attr->modify_hdr = mod_hdr;
1106         pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1107
1108         rule = mlx5_eswitch_add_offloaded_rule(esw, orig_spec, pre_ct_attr);
1109         if (IS_ERR(rule)) {
1110                 err = PTR_ERR(rule);
1111                 ct_dbg("Failed to add ct clear rule");
1112                 goto err_insert;
1113         }
1114
1115         attr->ct_attr.ct_flow = ct_flow;
1116         ct_flow->pre_ct_rule = rule;
1117         *flow_rule = rule;
1118
1119         return 0;
1120
1121 err_insert:
1122         mlx5_modify_header_dealloc(priv->mdev, mod_hdr);
1123 err_set_registers:
1124         netdev_warn(priv->netdev,
1125                     "Failed to offload ct clear flow, err %d\n", err);
1126         return err;
1127 }
1128
1129 struct mlx5_flow_handle *
1130 mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv,
1131                         struct mlx5e_tc_flow *flow,
1132                         struct mlx5_flow_spec *spec,
1133                         struct mlx5_esw_flow_attr *attr,
1134                         struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
1135 {
1136         bool clear_action = attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR;
1137         struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
1138         struct mlx5_flow_handle *rule;
1139         int err;
1140
1141         if (!ct_priv)
1142                 return ERR_PTR(-EOPNOTSUPP);
1143
1144         mutex_lock(&ct_priv->control_lock);
1145         if (clear_action)
1146                 err = __mlx5_tc_ct_flow_offload_clear(priv, flow, spec, attr,
1147                                                       mod_hdr_acts, &rule);
1148         else
1149                 err = __mlx5_tc_ct_flow_offload(priv, flow, spec, attr,
1150                                                 &rule);
1151         mutex_unlock(&ct_priv->control_lock);
1152         if (err)
1153                 return ERR_PTR(err);
1154
1155         return rule;
1156 }
1157
1158 static void
1159 __mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv,
1160                          struct mlx5_ct_flow *ct_flow)
1161 {
1162         struct mlx5_esw_flow_attr *pre_ct_attr = &ct_flow->pre_ct_attr;
1163         struct mlx5_eswitch *esw = ct_priv->esw;
1164
1165         mlx5_eswitch_del_offloaded_rule(esw, ct_flow->pre_ct_rule,
1166                                         pre_ct_attr);
1167         mlx5_modify_header_dealloc(esw->dev, pre_ct_attr->modify_hdr);
1168
1169         if (ct_flow->post_ct_rule) {
1170                 mlx5_eswitch_del_offloaded_rule(esw, ct_flow->post_ct_rule,
1171                                                 &ct_flow->post_ct_attr);
1172                 mlx5_esw_chains_put_chain_mapping(esw, ct_flow->chain_mapping);
1173                 idr_remove(&ct_priv->fte_ids, ct_flow->fte_id);
1174                 mlx5_tc_ct_del_ft_cb(ct_priv, ct_flow->ft);
1175         }
1176
1177         kfree(ct_flow);
1178 }
1179
1180 void
1181 mlx5_tc_ct_delete_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow,
1182                        struct mlx5_esw_flow_attr *attr)
1183 {
1184         struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
1185         struct mlx5_ct_flow *ct_flow = attr->ct_attr.ct_flow;
1186
1187         /* We are called on error to clean up stuff from parsing
1188          * but we don't have anything for now
1189          */
1190         if (!ct_flow)
1191                 return;
1192
1193         mutex_lock(&ct_priv->control_lock);
1194         __mlx5_tc_ct_delete_flow(ct_priv, ct_flow);
1195         mutex_unlock(&ct_priv->control_lock);
1196 }
1197
1198 static int
1199 mlx5_tc_ct_init_check_support(struct mlx5_eswitch *esw,
1200                               const char **err_msg)
1201 {
1202 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1203         /* cannot restore chain ID on HW miss */
1204
1205         *err_msg = "tc skb extension missing";
1206         return -EOPNOTSUPP;
1207 #endif
1208
1209         if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) {
1210                 *err_msg = "firmware level support is missing";
1211                 return -EOPNOTSUPP;
1212         }
1213
1214         if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) {
1215                 /* vlan workaround should be avoided for multi chain rules.
1216                  * This is just a sanity check as pop vlan action should
1217                  * be supported by any FW that supports ignore_flow_level
1218                  */
1219
1220                 *err_msg = "firmware vlan actions support is missing";
1221                 return -EOPNOTSUPP;
1222         }
1223
1224         if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev,
1225                                     fdb_modify_header_fwd_to_table)) {
1226                 /* CT always writes to registers which are mod header actions.
1227                  * Therefore, mod header and goto is required
1228                  */
1229
1230                 *err_msg = "firmware fwd and modify support is missing";
1231                 return -EOPNOTSUPP;
1232         }
1233
1234         if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1235                 *err_msg = "register loopback isn't supported";
1236                 return -EOPNOTSUPP;
1237         }
1238
1239         return 0;
1240 }
1241
1242 static void
1243 mlx5_tc_ct_init_err(struct mlx5e_rep_priv *rpriv, const char *msg, int err)
1244 {
1245         if (msg)
1246                 netdev_warn(rpriv->netdev,
1247                             "tc ct offload not supported, %s, err: %d\n",
1248                             msg, err);
1249         else
1250                 netdev_warn(rpriv->netdev,
1251                             "tc ct offload not supported, err: %d\n",
1252                             err);
1253 }
1254
1255 int
1256 mlx5_tc_ct_init(struct mlx5_rep_uplink_priv *uplink_priv)
1257 {
1258         struct mlx5_tc_ct_priv *ct_priv;
1259         struct mlx5e_rep_priv *rpriv;
1260         struct mlx5_eswitch *esw;
1261         struct mlx5e_priv *priv;
1262         const char *msg;
1263         int err;
1264
1265         rpriv = container_of(uplink_priv, struct mlx5e_rep_priv, uplink_priv);
1266         priv = netdev_priv(rpriv->netdev);
1267         esw = priv->mdev->priv.eswitch;
1268
1269         err = mlx5_tc_ct_init_check_support(esw, &msg);
1270         if (err) {
1271                 mlx5_tc_ct_init_err(rpriv, msg, err);
1272                 goto err_support;
1273         }
1274
1275         ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL);
1276         if (!ct_priv) {
1277                 mlx5_tc_ct_init_err(rpriv, NULL, -ENOMEM);
1278                 goto err_alloc;
1279         }
1280
1281         ct_priv->esw = esw;
1282         ct_priv->netdev = rpriv->netdev;
1283         ct_priv->ct = mlx5_esw_chains_create_global_table(esw);
1284         if (IS_ERR(ct_priv->ct)) {
1285                 err = PTR_ERR(ct_priv->ct);
1286                 mlx5_tc_ct_init_err(rpriv, "failed to create ct table", err);
1287                 goto err_ct_tbl;
1288         }
1289
1290         ct_priv->ct_nat = mlx5_esw_chains_create_global_table(esw);
1291         if (IS_ERR(ct_priv->ct_nat)) {
1292                 err = PTR_ERR(ct_priv->ct_nat);
1293                 mlx5_tc_ct_init_err(rpriv, "failed to create ct nat table",
1294                                     err);
1295                 goto err_ct_nat_tbl;
1296         }
1297
1298         ct_priv->post_ct = mlx5_esw_chains_create_global_table(esw);
1299         if (IS_ERR(ct_priv->post_ct)) {
1300                 err = PTR_ERR(ct_priv->post_ct);
1301                 mlx5_tc_ct_init_err(rpriv, "failed to create post ct table",
1302                                     err);
1303                 goto err_post_ct_tbl;
1304         }
1305
1306         idr_init(&ct_priv->fte_ids);
1307         idr_init(&ct_priv->tuple_ids);
1308         mutex_init(&ct_priv->control_lock);
1309         rhashtable_init(&ct_priv->zone_ht, &zone_params);
1310
1311         /* Done, set ct_priv to know it initializted */
1312         uplink_priv->ct_priv = ct_priv;
1313
1314         return 0;
1315
1316 err_post_ct_tbl:
1317         mlx5_esw_chains_destroy_global_table(esw, ct_priv->ct_nat);
1318 err_ct_nat_tbl:
1319         mlx5_esw_chains_destroy_global_table(esw, ct_priv->ct);
1320 err_ct_tbl:
1321         kfree(ct_priv);
1322 err_alloc:
1323 err_support:
1324
1325         return 0;
1326 }
1327
1328 void
1329 mlx5_tc_ct_clean(struct mlx5_rep_uplink_priv *uplink_priv)
1330 {
1331         struct mlx5_tc_ct_priv *ct_priv = uplink_priv->ct_priv;
1332
1333         if (!ct_priv)
1334                 return;
1335
1336         mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->post_ct);
1337         mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->ct_nat);
1338         mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->ct);
1339
1340         rhashtable_destroy(&ct_priv->zone_ht);
1341         mutex_destroy(&ct_priv->control_lock);
1342         idr_destroy(&ct_priv->tuple_ids);
1343         idr_destroy(&ct_priv->fte_ids);
1344         kfree(ct_priv);
1345
1346         uplink_priv->ct_priv = NULL;
1347 }
1348
1349 bool
1350 mlx5e_tc_ct_restore_flow(struct mlx5_rep_uplink_priv *uplink_priv,
1351                          struct sk_buff *skb, u32 tupleid)
1352 {
1353         struct mlx5_tc_ct_priv *ct_priv = uplink_priv->ct_priv;
1354         struct mlx5_ct_zone_rule *zone_rule;
1355         struct mlx5_ct_entry *entry;
1356
1357         if (!ct_priv || !tupleid)
1358                 return true;
1359
1360         zone_rule = idr_find(&ct_priv->tuple_ids, tupleid);
1361         if (!zone_rule)
1362                 return false;
1363
1364         entry = container_of(zone_rule, struct mlx5_ct_entry,
1365                              zone_rules[zone_rule->nat]);
1366         tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie);
1367
1368         return true;
1369 }