kernel/sched/isolation.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Housekeeping management. Manage the targets for routine code that can run on
   4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   5  *
   6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   8  *
   9  */
  10 #include "sched.h"
  11
  12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  13 EXPORT_SYMBOL_GPL(housekeeping_overridden);
  14 static cpumask_var_t housekeeping_mask;
  15 static unsigned int housekeeping_flags;
  16
  17 int housekeeping_any_cpu(enum hk_flags flags)
  18 {
  19         if (static_branch_unlikely(&housekeeping_overridden))
  20                 if (housekeeping_flags & flags)
  21                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  22         return smp_processor_id();
  23 }
  24 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  25
  26 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  27 {
  28         if (static_branch_unlikely(&housekeeping_overridden))
  29                 if (housekeeping_flags & flags)
  30                         return housekeeping_mask;
  31         return cpu_possible_mask;
  32 }
  33 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  34
  35 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  36 {
  37         if (static_branch_unlikely(&housekeeping_overridden))
  38                 if (housekeeping_flags & flags)
  39                         set_cpus_allowed_ptr(t, housekeeping_mask);
  40 }
  41 EXPORT_SYMBOL_GPL(housekeeping_affine);
  42
  43 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  44 {
  45         if (static_branch_unlikely(&housekeeping_overridden))
  46                 if (housekeeping_flags & flags)
  47                         return cpumask_test_cpu(cpu, housekeeping_mask);
  48         return true;
  49 }
  50 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  51
  52 void __init housekeeping_init(void)
  53 {
  54         if (!housekeeping_flags)
  55                 return;
  56
  57         static_branch_enable(&housekeeping_overridden);
  58
  59         if (housekeeping_flags & HK_FLAG_TICK)
  60                 sched_tick_offload_init();
  61
  62         /* We need at least one CPU to handle housekeeping work */
  63         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  64 }
  65
  66 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  67 {
  68         cpumask_var_t non_housekeeping_mask;
  69         cpumask_var_t tmp;
  70         int err;
  71
  72         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  73         err = cpulist_parse(str, non_housekeeping_mask);
  74         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  75                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  76                 free_bootmem_cpumask_var(non_housekeeping_mask);
  77                 return 0;
  78         }
  79
  80         alloc_bootmem_cpumask_var(&tmp);
  81         if (!housekeeping_flags) {
  82                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  83                 cpumask_andnot(housekeeping_mask,
  84                                cpu_possible_mask, non_housekeeping_mask);
  85
  86                 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
  87                 if (cpumask_empty(tmp)) {
  88                         pr_warn("Housekeeping: must include one present CPU, "
  89                                 "using boot CPU:%d\n", smp_processor_id());
  90                         __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  91                         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  92                 }
  93         } else {
  94                 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
  95                 if (cpumask_empty(tmp))
  96                         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  97                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  98                 if (!cpumask_equal(tmp, housekeeping_mask)) {
  99                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
 100                         free_bootmem_cpumask_var(tmp);
 101                         free_bootmem_cpumask_var(non_housekeeping_mask);
 102                         return 0;
 103                 }
 104         }
 105         free_bootmem_cpumask_var(tmp);
 106
 107         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 108                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 109                         tick_nohz_full_setup(non_housekeeping_mask);
 110                 } else {
 111                         pr_warn("Housekeeping: nohz unsupported."
 112                                 " Build with CONFIG_NO_HZ_FULL\n");
 113                         free_bootmem_cpumask_var(non_housekeeping_mask);
 114                         return 0;
 115                 }
 116         }
 117
 118         housekeeping_flags |= flags;
 119
 120         free_bootmem_cpumask_var(non_housekeeping_mask);
 121
 122         return 1;
 123 }
 124
 125 static int __init housekeeping_nohz_full_setup(char *str)
 126 {
 127         unsigned int flags;
 128
 129         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
 130
 131         return housekeeping_setup(str, flags);
 132 }
 133 __setup("nohz_full=", housekeeping_nohz_full_setup);
 134
 135 static int __init housekeeping_isolcpus_setup(char *str)
 136 {
 137         unsigned int flags = 0;
 138
 139         while (isalpha(*str)) {
 140                 if (!strncmp(str, "nohz,", 5)) {
 141                         str += 5;
 142                         flags |= HK_FLAG_TICK;
 143                         continue;
 144                 }
 145
 146                 if (!strncmp(str, "domain,", 7)) {
 147                         str += 7;
 148                         flags |= HK_FLAG_DOMAIN;
 149                         continue;
 150                 }
 151
 152                 pr_warn("isolcpus: Error, unknown flag\n");
 153                 return 0;
 154         }
 155
 156         /* Default behaviour for isolcpus without flags */
 157         if (!flags)
 158                 flags |= HK_FLAG_DOMAIN;
 159
 160         return housekeeping_setup(str, flags);
 161 }
 162 __setup("isolcpus=", housekeeping_isolcpus_setup);