OSDN Git Service

net/smc: use read semaphores to reduce unnecessary blocking in smc_buf_create() ...
authorD. Wythe <alibuda@linux.alibaba.com>
Thu, 2 Feb 2023 08:26:40 +0000 (16:26 +0800)
committerDavid S. Miller <davem@davemloft.net>
Sat, 4 Feb 2023 09:48:19 +0000 (09:48 +0000)
Following is part of Off-CPU graph during frequent SMC-R short-lived
processing:

process_one_work (51.19%)
smc_close_passive_work (28.36%)
smcr_buf_unuse (28.34%)
rwsem_down_write_slowpath (28.22%)

smc_listen_work (22.83%)
smc_clc_wait_msg (1.84%)
smc_buf_create (20.45%)
smcr_buf_map_usable_links
rwsem_down_write_slowpath (20.43%)
smcr_lgr_reg_rmbs (0.53%)
rwsem_down_write_slowpath (0.43%)
smc_llc_do_confirm_rkey (0.08%)

We can clearly see that during the connection establishment time,
waiting time of connections is not on IO, but on llc_conf_mutex.

What is more important, the core critical area (smcr_buf_unuse() &
smc_buf_create()) only perfroms read semantics on links, we can
easily replace it with read semaphore.

Signed-off-by: D. Wythe <alibuda@linux.alibaba.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
net/smc/smc_core.c

index 22ddce6..10a9e09 100644 (file)
@@ -1106,10 +1106,10 @@ static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb,
                rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
                if (!rc) {
                        /* protect against smc_llc_cli_rkey_exchange() */
-                       down_write(&lgr->llc_conf_mutex);
+                       down_read(&lgr->llc_conf_mutex);
                        smc_llc_do_delete_rkey(lgr, buf_desc);
                        buf_desc->is_conf_rkey = false;
-                       up_write(&lgr->llc_conf_mutex);
+                       up_read(&lgr->llc_conf_mutex);
                        smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
                }
        }
@@ -2247,7 +2247,7 @@ static int smcr_buf_map_usable_links(struct smc_link_group *lgr,
        int i, rc = 0, cnt = 0;
 
        /* protect against parallel link reconfiguration */
-       down_write(&lgr->llc_conf_mutex);
+       down_read(&lgr->llc_conf_mutex);
        for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
                struct smc_link *lnk = &lgr->lnk[i];
 
@@ -2260,7 +2260,7 @@ static int smcr_buf_map_usable_links(struct smc_link_group *lgr,
                cnt++;
        }
 out:
-       up_write(&lgr->llc_conf_mutex);
+       up_read(&lgr->llc_conf_mutex);
        if (!rc && !cnt)
                rc = -EINVAL;
        return rc;