drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 static bool suitable_policy(struct cpufreq_policy *policy, bool active)
  42 {
  43         return active == !policy_is_inactive(policy);
  44 }
  45
  46 /* Finds Next Acive/Inactive policy */
  47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
  48                                           bool active)
  49 {
  50         do {
  51                 policy = list_next_entry(policy, policy_list);
  52
  53                 /* No more policies in the list */
  54                 if (&policy->policy_list == &cpufreq_policy_list)
  55                         return NULL;
  56         } while (!suitable_policy(policy, active));
  57
  58         return policy;
  59 }
  60
  61 static struct cpufreq_policy *first_policy(bool active)
  62 {
  63         struct cpufreq_policy *policy;
  64
  65         /* No policies in the list */
  66         if (list_empty(&cpufreq_policy_list))
  67                 return NULL;
  68
  69         policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
  70                                   policy_list);
  71
  72         if (!suitable_policy(policy, active))
  73                 policy = next_policy(policy, active);
  74
  75         return policy;
  76 }
  77
  78 /* Macros to iterate over CPU policies */
  79 #define for_each_suitable_policy(__policy, __active)    \
  80         for (__policy = first_policy(__active);         \
  81              __policy;                                  \
  82              __policy = next_policy(__policy, __active))
  83
  84 #define for_each_active_policy(__policy)                \
  85         for_each_suitable_policy(__policy, true)
  86 #define for_each_inactive_policy(__policy)              \
  87         for_each_suitable_policy(__policy, false)
  88
  89 #define for_each_policy(__policy)                       \
  90         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  91
  92 /* Iterate over governors */
  93 static LIST_HEAD(cpufreq_governor_list);
  94 #define for_each_governor(__governor)                           \
  95         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  96
  97 /**
  98  * The "cpufreq driver" - the arch- or hardware-dependent low
  99  * level driver of CPUFreq support, and its spinlock. This lock
 100  * also protects the cpufreq_cpu_data array.
 101  */
 102 static struct cpufreq_driver *cpufreq_driver;
 103 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
 104 static DEFINE_RWLOCK(cpufreq_driver_lock);
 105 DEFINE_MUTEX(cpufreq_governor_lock);
 106
 107 /* Flag to suspend/resume CPUFreq governors */
 108 static bool cpufreq_suspended;
 109
 110 static inline bool has_target(void)
 111 {
 112         return cpufreq_driver->target_index || cpufreq_driver->target;
 113 }
 114
 115 /* internal prototypes */
 116 static int __cpufreq_governor(struct cpufreq_policy *policy,
 117                 unsigned int event);
 118 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 119 static void handle_update(struct work_struct *work);
 120
 121 /**
 122  * Two notifier lists: the "policy" list is involved in the
 123  * validation process for a new CPU frequency policy; the
 124  * "transition" list for kernel code that needs to handle
 125  * changes to devices when the CPU clock speed changes.
 126  * The mutex locks both lists.
 127  */
 128 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 129 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 130
 131 static bool init_cpufreq_transition_notifier_list_called;
 132 static int __init init_cpufreq_transition_notifier_list(void)
 133 {
 134         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 135         init_cpufreq_transition_notifier_list_called = true;
 136         return 0;
 137 }
 138 pure_initcall(init_cpufreq_transition_notifier_list);
 139
 140 static int off __read_mostly;
 141 static int cpufreq_disabled(void)
 142 {
 143         return off;
 144 }
 145 void disable_cpufreq(void)
 146 {
 147         off = 1;
 148 }
 149 static DEFINE_MUTEX(cpufreq_governor_mutex);
 150
 151 bool have_governor_per_policy(void)
 152 {
 153         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 154 }
 155 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 156
 157 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 158 {
 159         if (have_governor_per_policy())
 160                 return &policy->kobj;
 161         else
 162                 return cpufreq_global_kobject;
 163 }
 164 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 165
 166 struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
 167 {
 168         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 169
 170         return policy && !policy_is_inactive(policy) ?
 171                 policy->freq_table : NULL;
 172 }
 173 EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);
 174
 175 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 176 {
 177         u64 idle_time;
 178         u64 cur_wall_time;
 179         u64 busy_time;
 180
 181         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 182
 183         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 184         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 185         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 186         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 187         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 188         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 189
 190         idle_time = cur_wall_time - busy_time;
 191         if (wall)
 192                 *wall = cputime_to_usecs(cur_wall_time);
 193
 194         return cputime_to_usecs(idle_time);
 195 }
 196
 197 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 198 {
 199         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 200
 201         if (idle_time == -1ULL)
 202                 return get_cpu_idle_time_jiffy(cpu, wall);
 203         else if (!io_busy)
 204                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 205
 206         return idle_time;
 207 }
 208 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 209
 210 /*
 211  * This is a generic cpufreq init() routine which can be used by cpufreq
 212  * drivers of SMP systems. It will do following:
 213  * - validate & show freq table passed
 214  * - set policies transition latency
 215  * - policy->cpus with all possible CPUs
 216  */
 217 int cpufreq_generic_init(struct cpufreq_policy *policy,
 218                 struct cpufreq_frequency_table *table,
 219                 unsigned int transition_latency)
 220 {
 221         int ret;
 222
 223         ret = cpufreq_table_validate_and_show(policy, table);
 224         if (ret) {
 225                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 226                 return ret;
 227         }
 228
 229         policy->cpuinfo.transition_latency = transition_latency;
 230
 231         /*
 232          * The driver only supports the SMP configuration where all processors
 233          * share the clock and voltage and clock.
 234          */
 235         cpumask_setall(policy->cpus);
 236
 237         return 0;
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 240
 241 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 242 {
 243         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 244
 245         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 246 }
 247 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
 248
 249 unsigned int cpufreq_generic_get(unsigned int cpu)
 250 {
 251         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 252
 253         if (!policy || IS_ERR(policy->clk)) {
 254                 pr_err("%s: No %s associated to cpu: %d\n",
 255                        __func__, policy ? "clk" : "policy", cpu);
 256                 return 0;
 257         }
 258
 259         return clk_get_rate(policy->clk) / 1000;
 260 }
 261 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 262
 263 /**
 264  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 265  *
 266  * @cpu: cpu to find policy for.
 267  *
 268  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 269  * It also increments the kobject reference count to mark it busy and so would
 270  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 271  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 272  * freed as that depends on the kobj count.
 273  *
 274  * Return: A valid policy on success, otherwise NULL on failure.
 275  */
 276 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 277 {
 278         struct cpufreq_policy *policy = NULL;
 279         unsigned long flags;
 280
 281         if (WARN_ON(cpu >= nr_cpu_ids))
 282                 return NULL;
 283
 284         /* get the cpufreq driver */
 285         read_lock_irqsave(&cpufreq_driver_lock, flags);
 286
 287         if (cpufreq_driver) {
 288                 /* get the CPU */
 289                 policy = cpufreq_cpu_get_raw(cpu);
 290                 if (policy)
 291                         kobject_get(&policy->kobj);
 292         }
 293
 294         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 295
 296         return policy;
 297 }
 298 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 299
 300 /**
 301  * cpufreq_cpu_put: Decrements the usage count of a policy
 302  *
 303  * @policy: policy earlier returned by cpufreq_cpu_get().
 304  *
 305  * This decrements the kobject reference count incremented earlier by calling
 306  * cpufreq_cpu_get().
 307  */
 308 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 309 {
 310         kobject_put(&policy->kobj);
 311 }
 312 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 313
 314 /*********************************************************************
 315  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 316  *********************************************************************/
 317
 318 /**
 319  * adjust_jiffies - adjust the system "loops_per_jiffy"
 320  *
 321  * This function alters the system "loops_per_jiffy" for the clock
 322  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 323  * systems as each CPU might be scaled differently. So, use the arch
 324  * per-CPU loops_per_jiffy value wherever possible.
 325  */
 326 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 327 {
 328 #ifndef CONFIG_SMP
 329         static unsigned long l_p_j_ref;
 330         static unsigned int l_p_j_ref_freq;
 331
 332         if (ci->flags & CPUFREQ_CONST_LOOPS)
 333                 return;
 334
 335         if (!l_p_j_ref_freq) {
 336                 l_p_j_ref = loops_per_jiffy;
 337                 l_p_j_ref_freq = ci->old;
 338                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 339                          l_p_j_ref, l_p_j_ref_freq);
 340         }
 341         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 342                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 343                                                                 ci->new);
 344                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 345                          loops_per_jiffy, ci->new);
 346         }
 347 #endif
 348 }
 349
 350 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 351                 struct cpufreq_freqs *freqs, unsigned int state)
 352 {
 353         BUG_ON(irqs_disabled());
 354
 355         if (cpufreq_disabled())
 356                 return;
 357
 358         freqs->flags = cpufreq_driver->flags;
 359         pr_debug("notification %u of frequency transition to %u kHz\n",
 360                  state, freqs->new);
 361
 362         switch (state) {
 363
 364         case CPUFREQ_PRECHANGE:
 365                 /* detect if the driver reported a value as "old frequency"
 366                  * which is not equal to what the cpufreq core thinks is
 367                  * "old frequency".
 368                  */
 369                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 370                         if ((policy) && (policy->cpu == freqs->cpu) &&
 371                             (policy->cur) && (policy->cur != freqs->old)) {
 372                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 373                                          freqs->old, policy->cur);
 374                                 freqs->old = policy->cur;
 375                         }
 376                 }
 377                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 378                                 CPUFREQ_PRECHANGE, freqs);
 379                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 380                 break;
 381
 382         case CPUFREQ_POSTCHANGE:
 383                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 384                 pr_debug("FREQ: %lu - CPU: %lu\n",
 385                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 386                 trace_cpu_frequency(freqs->new, freqs->cpu);
 387                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 388                                 CPUFREQ_POSTCHANGE, freqs);
 389                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 390                         policy->cur = freqs->new;
 391                 break;
 392         }
 393 }
 394
 395 /**
 396  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 397  * on frequency transition.
 398  *
 399  * This function calls the transition notifiers and the "adjust_jiffies"
 400  * function. It is called twice on all CPU frequency changes that have
 401  * external effects.
 402  */
 403 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 404                 struct cpufreq_freqs *freqs, unsigned int state)
 405 {
 406         for_each_cpu(freqs->cpu, policy->cpus)
 407                 __cpufreq_notify_transition(policy, freqs, state);
 408 }
 409
 410 /* Do post notifications when there are chances that transition has failed */
 411 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 412                 struct cpufreq_freqs *freqs, int transition_failed)
 413 {
 414         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 415         if (!transition_failed)
 416                 return;
 417
 418         swap(freqs->old, freqs->new);
 419         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 420         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 421 }
 422
 423 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 424                 struct cpufreq_freqs *freqs)
 425 {
 426
 427         /*
 428          * Catch double invocations of _begin() which lead to self-deadlock.
 429          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 430          * doesn't invoke _begin() on their behalf, and hence the chances of
 431          * double invocations are very low. Moreover, there are scenarios
 432          * where these checks can emit false-positive warnings in these
 433          * drivers; so we avoid that by skipping them altogether.
 434          */
 435         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 436                                 && current == policy->transition_task);
 437
 438 wait:
 439         wait_event(policy->transition_wait, !policy->transition_ongoing);
 440
 441         spin_lock(&policy->transition_lock);
 442
 443         if (unlikely(policy->transition_ongoing)) {
 444                 spin_unlock(&policy->transition_lock);
 445                 goto wait;
 446         }
 447
 448         policy->transition_ongoing = true;
 449         policy->transition_task = current;
 450
 451         spin_unlock(&policy->transition_lock);
 452
 453         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 454 }
 455 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 456
 457 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 458                 struct cpufreq_freqs *freqs, int transition_failed)
 459 {
 460         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 461                 return;
 462
 463         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 464
 465         policy->transition_ongoing = false;
 466         policy->transition_task = NULL;
 467
 468         wake_up(&policy->transition_wait);
 469 }
 470 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 471
 472
 473 /*********************************************************************
 474  *                          SYSFS INTERFACE                          *
 475  *********************************************************************/
 476 static ssize_t show_boost(struct kobject *kobj,
 477                                  struct attribute *attr, char *buf)
 478 {
 479         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 480 }
 481
 482 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 483                                   const char *buf, size_t count)
 484 {
 485         int ret, enable;
 486
 487         ret = sscanf(buf, "%d", &enable);
 488         if (ret != 1 || enable < 0 || enable > 1)
 489                 return -EINVAL;
 490
 491         if (cpufreq_boost_trigger_state(enable)) {
 492                 pr_err("%s: Cannot %s BOOST!\n",
 493                        __func__, enable ? "enable" : "disable");
 494                 return -EINVAL;
 495         }
 496
 497         pr_debug("%s: cpufreq BOOST %s\n",
 498                  __func__, enable ? "enabled" : "disabled");
 499
 500         return count;
 501 }
 502 define_one_global_rw(boost);
 503
 504 static struct cpufreq_governor *find_governor(const char *str_governor)
 505 {
 506         struct cpufreq_governor *t;
 507
 508         for_each_governor(t)
 509                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 510                         return t;
 511
 512         return NULL;
 513 }
 514
 515 /**
 516  * cpufreq_parse_governor - parse a governor string
 517  */
 518 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 519                                 struct cpufreq_governor **governor)
 520 {
 521         int err = -EINVAL;
 522
 523         if (cpufreq_driver->setpolicy) {
 524                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 525                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 526                         err = 0;
 527                 } else if (!strncasecmp(str_governor, "powersave",
 528                                                 CPUFREQ_NAME_LEN)) {
 529                         *policy = CPUFREQ_POLICY_POWERSAVE;
 530                         err = 0;
 531                 }
 532         } else {
 533                 struct cpufreq_governor *t;
 534
 535                 mutex_lock(&cpufreq_governor_mutex);
 536
 537                 t = find_governor(str_governor);
 538
 539                 if (t == NULL) {
 540                         int ret;
 541
 542                         mutex_unlock(&cpufreq_governor_mutex);
 543                         ret = request_module("cpufreq_%s", str_governor);
 544                         mutex_lock(&cpufreq_governor_mutex);
 545
 546                         if (ret == 0)
 547                                 t = find_governor(str_governor);
 548                 }
 549
 550                 if (t != NULL) {
 551                         *governor = t;
 552                         err = 0;
 553                 }
 554                 if (t && !try_module_get(t->owner))
 555                         t = NULL;
 556
 557                 mutex_unlock(&cpufreq_governor_mutex);
 558         }
 559         return err;
 560 }
 561
 562 /**
 563  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 564  * print out cpufreq information
 565  *
 566  * Write out information from cpufreq_driver->policy[cpu]; object must be
 567  * "unsigned int".
 568  */
 569
 570 #define show_one(file_name, object)                     \
 571 static ssize_t show_##file_name                         \
 572 (struct cpufreq_policy *policy, char *buf)              \
 573 {                                                       \
 574         return sprintf(buf, "%u\n", policy->object);    \
 575 }
 576
 577 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 578 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 579 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 580 show_one(scaling_min_freq, min);
 581 show_one(scaling_max_freq, max);
 582
 583 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 584 {
 585         ssize_t ret;
 586
 587         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 588                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 589         else
 590                 ret = sprintf(buf, "%u\n", policy->cur);
 591         return ret;
 592 }
 593
 594 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 595                                 struct cpufreq_policy *new_policy);
 596
 597 /**
 598  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 599  */
 600 #define store_one(file_name, object)                    \
 601 static ssize_t store_##file_name                                        \
 602 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 603 {                                                                       \
 604         int ret, temp;                                                  \
 605         struct cpufreq_policy new_policy;                               \
 606                                                                         \
 607         memcpy(&new_policy, policy, sizeof(*policy));                   \
 608                                                                         \
 609         ret = sscanf(buf, "%u", &new_policy.object);                    \
 610         if (ret != 1)                                                   \
 611                 return -EINVAL;                                         \
 612                                                                         \
 613         temp = new_policy.object;                                       \
 614         ret = cpufreq_set_policy(policy, &new_policy);          \
 615         if (!ret)                                                       \
 616                 policy->user_policy.object = temp;                      \
 617                                                                         \
 618         return ret ? ret : count;                                       \
 619 }
 620
 621 store_one(scaling_min_freq, min);
 622 store_one(scaling_max_freq, max);
 623
 624 /**
 625  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 626  */
 627 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 628                                         char *buf)
 629 {
 630         unsigned int cur_freq = __cpufreq_get(policy);
 631
 632         if (cur_freq)
 633                 return sprintf(buf, "%u\n", cur_freq);
 634
 635         return sprintf(buf, "<unknown>\n");
 636 }
 637
 638 /**
 639  * show_scaling_governor - show the current policy for the specified CPU
 640  */
 641 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 642 {
 643         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 644                 return sprintf(buf, "powersave\n");
 645         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 646                 return sprintf(buf, "performance\n");
 647         else if (policy->governor)
 648                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 649                                 policy->governor->name);
 650         return -EINVAL;
 651 }
 652
 653 /**
 654  * store_scaling_governor - store policy for the specified CPU
 655  */
 656 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 657                                         const char *buf, size_t count)
 658 {
 659         int ret;
 660         char    str_governor[16];
 661         struct cpufreq_policy new_policy;
 662
 663         memcpy(&new_policy, policy, sizeof(*policy));
 664
 665         ret = sscanf(buf, "%15s", str_governor);
 666         if (ret != 1)
 667                 return -EINVAL;
 668
 669         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 670                                                 &new_policy.governor))
 671                 return -EINVAL;
 672
 673         ret = cpufreq_set_policy(policy, &new_policy);
 674
 675         if (new_policy.governor)
 676                 module_put(new_policy.governor->owner);
 677
 678         return ret ? ret : count;
 679 }
 680
 681 /**
 682  * show_scaling_driver - show the cpufreq driver currently loaded
 683  */
 684 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 685 {
 686         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 687 }
 688
 689 /**
 690  * show_scaling_available_governors - show the available CPUfreq governors
 691  */
 692 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 693                                                 char *buf)
 694 {
 695         ssize_t i = 0;
 696         struct cpufreq_governor *t;
 697
 698         if (!has_target()) {
 699                 i += sprintf(buf, "performance powersave");
 700                 goto out;
 701         }
 702
 703         for_each_governor(t) {
 704                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 705                     - (CPUFREQ_NAME_LEN + 2)))
 706                         goto out;
 707                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 708         }
 709 out:
 710         i += sprintf(&buf[i], "\n");
 711         return i;
 712 }
 713
 714 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 715 {
 716         ssize_t i = 0;
 717         unsigned int cpu;
 718
 719         for_each_cpu(cpu, mask) {
 720                 if (i)
 721                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 722                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 723                 if (i >= (PAGE_SIZE - 5))
 724                         break;
 725         }
 726         i += sprintf(&buf[i], "\n");
 727         return i;
 728 }
 729 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 730
 731 /**
 732  * show_related_cpus - show the CPUs affected by each transition even if
 733  * hw coordination is in use
 734  */
 735 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 736 {
 737         return cpufreq_show_cpus(policy->related_cpus, buf);
 738 }
 739
 740 /**
 741  * show_affected_cpus - show the CPUs affected by each transition
 742  */
 743 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 744 {
 745         return cpufreq_show_cpus(policy->cpus, buf);
 746 }
 747
 748 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 749                                         const char *buf, size_t count)
 750 {
 751         unsigned int freq = 0;
 752         unsigned int ret;
 753
 754         if (!policy->governor || !policy->governor->store_setspeed)
 755                 return -EINVAL;
 756
 757         ret = sscanf(buf, "%u", &freq);
 758         if (ret != 1)
 759                 return -EINVAL;
 760
 761         policy->governor->store_setspeed(policy, freq);
 762
 763         return count;
 764 }
 765
 766 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 767 {
 768         if (!policy->governor || !policy->governor->show_setspeed)
 769                 return sprintf(buf, "<unsupported>\n");
 770
 771         return policy->governor->show_setspeed(policy, buf);
 772 }
 773
 774 /**
 775  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 776  */
 777 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 778 {
 779         unsigned int limit;
 780         int ret;
 781         if (cpufreq_driver->bios_limit) {
 782                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 783                 if (!ret)
 784                         return sprintf(buf, "%u\n", limit);
 785         }
 786         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 787 }
 788
 789 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 790 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 791 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 792 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 793 cpufreq_freq_attr_ro(scaling_available_governors);
 794 cpufreq_freq_attr_ro(scaling_driver);
 795 cpufreq_freq_attr_ro(scaling_cur_freq);
 796 cpufreq_freq_attr_ro(bios_limit);
 797 cpufreq_freq_attr_ro(related_cpus);
 798 cpufreq_freq_attr_ro(affected_cpus);
 799 cpufreq_freq_attr_rw(scaling_min_freq);
 800 cpufreq_freq_attr_rw(scaling_max_freq);
 801 cpufreq_freq_attr_rw(scaling_governor);
 802 cpufreq_freq_attr_rw(scaling_setspeed);
 803
 804 static struct attribute *default_attrs[] = {
 805         &cpuinfo_min_freq.attr,
 806         &cpuinfo_max_freq.attr,
 807         &cpuinfo_transition_latency.attr,
 808         &scaling_min_freq.attr,
 809         &scaling_max_freq.attr,
 810         &affected_cpus.attr,
 811         &related_cpus.attr,
 812         &scaling_governor.attr,
 813         &scaling_driver.attr,
 814         &scaling_available_governors.attr,
 815         &scaling_setspeed.attr,
 816         NULL
 817 };
 818
 819 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 820 #define to_attr(a) container_of(a, struct freq_attr, attr)
 821
 822 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 823 {
 824         struct cpufreq_policy *policy = to_policy(kobj);
 825         struct freq_attr *fattr = to_attr(attr);
 826         ssize_t ret;
 827
 828         down_read(&policy->rwsem);
 829
 830         if (fattr->show)
 831                 ret = fattr->show(policy, buf);
 832         else
 833                 ret = -EIO;
 834
 835         up_read(&policy->rwsem);
 836
 837         return ret;
 838 }
 839
 840 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 841                      const char *buf, size_t count)
 842 {
 843         struct cpufreq_policy *policy = to_policy(kobj);
 844         struct freq_attr *fattr = to_attr(attr);
 845         ssize_t ret = -EINVAL;
 846
 847         get_online_cpus();
 848
 849         if (!cpu_online(policy->cpu))
 850                 goto unlock;
 851
 852         down_write(&policy->rwsem);
 853
 854         if (fattr->store)
 855                 ret = fattr->store(policy, buf, count);
 856         else
 857                 ret = -EIO;
 858
 859         up_write(&policy->rwsem);
 860 unlock:
 861         put_online_cpus();
 862
 863         return ret;
 864 }
 865
 866 static void cpufreq_sysfs_release(struct kobject *kobj)
 867 {
 868         struct cpufreq_policy *policy = to_policy(kobj);
 869         pr_debug("last reference is dropped\n");
 870         complete(&policy->kobj_unregister);
 871 }
 872
 873 static const struct sysfs_ops sysfs_ops = {
 874         .show   = show,
 875         .store  = store,
 876 };
 877
 878 static struct kobj_type ktype_cpufreq = {
 879         .sysfs_ops      = &sysfs_ops,
 880         .default_attrs  = default_attrs,
 881         .release        = cpufreq_sysfs_release,
 882 };
 883
 884 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 885 {
 886         struct device *cpu_dev;
 887
 888         pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
 889
 890         if (!policy)
 891                 return 0;
 892
 893         cpu_dev = get_cpu_device(cpu);
 894         if (WARN_ON(!cpu_dev))
 895                 return 0;
 896
 897         return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
 898 }
 899
 900 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 901 {
 902         struct device *cpu_dev;
 903
 904         pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
 905
 906         cpu_dev = get_cpu_device(cpu);
 907         if (WARN_ON(!cpu_dev))
 908                 return;
 909
 910         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
 911 }
 912
 913 /* Add/remove symlinks for all related CPUs */
 914 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 915 {
 916         unsigned int j;
 917         int ret = 0;
 918
 919         /* Some related CPUs might not be present (physically hotplugged) */
 920         for_each_cpu(j, policy->real_cpus) {
 921                 ret = add_cpu_dev_symlink(policy, j);
 922                 if (ret)
 923                         break;
 924         }
 925
 926         return ret;
 927 }
 928
 929 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
 930 {
 931         unsigned int j;
 932
 933         /* Some related CPUs might not be present (physically hotplugged) */
 934         for_each_cpu(j, policy->real_cpus)
 935                 remove_cpu_dev_symlink(policy, j);
 936 }
 937
 938 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
 939 {
 940         struct freq_attr **drv_attr;
 941         int ret = 0;
 942
 943         /* set up files for this cpu device */
 944         drv_attr = cpufreq_driver->attr;
 945         while (drv_attr && *drv_attr) {
 946                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 947                 if (ret)
 948                         return ret;
 949                 drv_attr++;
 950         }
 951         if (cpufreq_driver->get) {
 952                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 953                 if (ret)
 954                         return ret;
 955         }
 956
 957         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 958         if (ret)
 959                 return ret;
 960
 961         if (cpufreq_driver->bios_limit) {
 962                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 963                 if (ret)
 964                         return ret;
 965         }
 966
 967         return cpufreq_add_dev_symlink(policy);
 968 }
 969
 970 static int cpufreq_init_policy(struct cpufreq_policy *policy)
 971 {
 972         struct cpufreq_governor *gov = NULL;
 973         struct cpufreq_policy new_policy;
 974
 975         memcpy(&new_policy, policy, sizeof(*policy));
 976
 977         /* Update governor of new_policy to the governor used before hotplug */
 978         gov = find_governor(policy->last_governor);
 979         if (gov)
 980                 pr_debug("Restoring governor %s for cpu %d\n",
 981                                 policy->governor->name, policy->cpu);
 982         else
 983                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 984
 985         new_policy.governor = gov;
 986
 987         /* Use the default policy if there is no last_policy. */
 988         if (cpufreq_driver->setpolicy) {
 989                 if (policy->last_policy)
 990                         new_policy.policy = policy->last_policy;
 991                 else
 992                         cpufreq_parse_governor(gov->name, &new_policy.policy,
 993                                                NULL);
 994         }
 995         /* set default policy */
 996         return cpufreq_set_policy(policy, &new_policy);
 997 }
 998
 999 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1000 {
1001         int ret = 0;
1002
1003         /* Has this CPU been taken care of already? */
1004         if (cpumask_test_cpu(cpu, policy->cpus))
1005                 return 0;
1006
1007         if (has_target()) {
1008                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1009                 if (ret) {
1010                         pr_err("%s: Failed to stop governor\n", __func__);
1011                         return ret;
1012                 }
1013         }
1014
1015         down_write(&policy->rwsem);
1016         cpumask_set_cpu(cpu, policy->cpus);
1017         up_write(&policy->rwsem);
1018
1019         if (has_target()) {
1020                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1021                 if (!ret)
1022                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1023
1024                 if (ret) {
1025                         pr_err("%s: Failed to start governor\n", __func__);
1026                         return ret;
1027                 }
1028         }
1029
1030         return 0;
1031 }
1032
1033 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1034 {
1035         struct device *dev = get_cpu_device(cpu);
1036         struct cpufreq_policy *policy;
1037
1038         if (WARN_ON(!dev))
1039                 return NULL;
1040
1041         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1042         if (!policy)
1043                 return NULL;
1044
1045         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1046                 goto err_free_policy;
1047
1048         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1049                 goto err_free_cpumask;
1050
1051         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1052                 goto err_free_rcpumask;
1053
1054         kobject_init(&policy->kobj, &ktype_cpufreq);
1055         INIT_LIST_HEAD(&policy->policy_list);
1056         init_rwsem(&policy->rwsem);
1057         spin_lock_init(&policy->transition_lock);
1058         init_waitqueue_head(&policy->transition_wait);
1059         init_completion(&policy->kobj_unregister);
1060         INIT_WORK(&policy->update, handle_update);
1061
1062         policy->cpu = cpu;
1063         return policy;
1064
1065 err_free_rcpumask:
1066         free_cpumask_var(policy->related_cpus);
1067 err_free_cpumask:
1068         free_cpumask_var(policy->cpus);
1069 err_free_policy:
1070         kfree(policy);
1071
1072         return NULL;
1073 }
1074
1075 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1076 {
1077         struct kobject *kobj;
1078         struct completion *cmp;
1079
1080         if (notify)
1081                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1082                                              CPUFREQ_REMOVE_POLICY, policy);
1083
1084         down_write(&policy->rwsem);
1085         cpufreq_remove_dev_symlink(policy);
1086         kobj = &policy->kobj;
1087         cmp = &policy->kobj_unregister;
1088         up_write(&policy->rwsem);
1089         kobject_put(kobj);
1090
1091         /*
1092          * We need to make sure that the underlying kobj is
1093          * actually not referenced anymore by anybody before we
1094          * proceed with unloading.
1095          */
1096         pr_debug("waiting for dropping of refcount\n");
1097         wait_for_completion(cmp);
1098         pr_debug("wait complete\n");
1099 }
1100
1101 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1102 {
1103         unsigned long flags;
1104         int cpu;
1105
1106         /* Remove policy from list */
1107         write_lock_irqsave(&cpufreq_driver_lock, flags);
1108         list_del(&policy->policy_list);
1109
1110         for_each_cpu(cpu, policy->related_cpus)
1111                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1112         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1113
1114         cpufreq_policy_put_kobj(policy, notify);
1115         free_cpumask_var(policy->real_cpus);
1116         free_cpumask_var(policy->related_cpus);
1117         free_cpumask_var(policy->cpus);
1118         kfree(policy);
1119 }
1120
1121 static int cpufreq_online(unsigned int cpu)
1122 {
1123         struct cpufreq_policy *policy;
1124         bool new_policy;
1125         unsigned long flags;
1126         unsigned int j;
1127         int ret;
1128
1129         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1130
1131         /* Check if this CPU already has a policy to manage it */
1132         policy = per_cpu(cpufreq_cpu_data, cpu);
1133         if (policy) {
1134                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1135                 if (!policy_is_inactive(policy))
1136                         return cpufreq_add_policy_cpu(policy, cpu);
1137
1138                 /* This is the only online CPU for the policy.  Start over. */
1139                 new_policy = false;
1140                 down_write(&policy->rwsem);
1141                 policy->cpu = cpu;
1142                 policy->governor = NULL;
1143                 up_write(&policy->rwsem);
1144         } else {
1145                 new_policy = true;
1146                 policy = cpufreq_policy_alloc(cpu);
1147                 if (!policy)
1148                         return -ENOMEM;
1149         }
1150
1151         cpumask_copy(policy->cpus, cpumask_of(cpu));
1152
1153         /* call driver. From then on the cpufreq must be able
1154          * to accept all calls to ->verify and ->setpolicy for this CPU
1155          */
1156         ret = cpufreq_driver->init(policy);
1157         if (ret) {
1158                 pr_debug("initialization failed\n");
1159                 goto out_free_policy;
1160         }
1161
1162         down_write(&policy->rwsem);
1163
1164         if (new_policy) {
1165                 /* related_cpus should at least include policy->cpus. */
1166                 cpumask_copy(policy->related_cpus, policy->cpus);
1167                 /* Remember CPUs present at the policy creation time. */
1168                 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
1169
1170                 /* Name and add the kobject */
1171                 ret = kobject_add(&policy->kobj, cpufreq_global_kobject,
1172                                   "policy%u",
1173                                   cpumask_first(policy->related_cpus));
1174                 if (ret) {
1175                         pr_err("%s: failed to add policy->kobj: %d\n", __func__,
1176                                ret);
1177                         goto out_exit_policy;
1178                 }
1179         }
1180
1181         /*
1182          * affected cpus must always be the one, which are online. We aren't
1183          * managing offline cpus here.
1184          */
1185         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1186
1187         if (new_policy) {
1188                 policy->user_policy.min = policy->min;
1189                 policy->user_policy.max = policy->max;
1190
1191                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1192                 for_each_cpu(j, policy->related_cpus)
1193                         per_cpu(cpufreq_cpu_data, j) = policy;
1194                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1195         } else {
1196                 policy->min = policy->user_policy.min;
1197                 policy->max = policy->user_policy.max;
1198         }
1199
1200         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1201                 policy->cur = cpufreq_driver->get(policy->cpu);
1202                 if (!policy->cur) {
1203                         pr_err("%s: ->get() failed\n", __func__);
1204                         goto out_exit_policy;
1205                 }
1206         }
1207
1208         /*
1209          * Sometimes boot loaders set CPU frequency to a value outside of
1210          * frequency table present with cpufreq core. In such cases CPU might be
1211          * unstable if it has to run on that frequency for long duration of time
1212          * and so its better to set it to a frequency which is specified in
1213          * freq-table. This also makes cpufreq stats inconsistent as
1214          * cpufreq-stats would fail to register because current frequency of CPU
1215          * isn't found in freq-table.
1216          *
1217          * Because we don't want this change to effect boot process badly, we go
1218          * for the next freq which is >= policy->cur ('cur' must be set by now,
1219          * otherwise we will end up setting freq to lowest of the table as 'cur'
1220          * is initialized to zero).
1221          *
1222          * We are passing target-freq as "policy->cur - 1" otherwise
1223          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1224          * equal to target-freq.
1225          */
1226         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1227             && has_target()) {
1228                 /* Are we running at unknown frequency ? */
1229                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1230                 if (ret == -EINVAL) {
1231                         /* Warn user and fix it */
1232                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1233                                 __func__, policy->cpu, policy->cur);
1234                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1235                                 CPUFREQ_RELATION_L);
1236
1237                         /*
1238                          * Reaching here after boot in a few seconds may not
1239                          * mean that system will remain stable at "unknown"
1240                          * frequency for longer duration. Hence, a BUG_ON().
1241                          */
1242                         BUG_ON(ret);
1243                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1244                                 __func__, policy->cpu, policy->cur);
1245                 }
1246         }
1247
1248         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1249                                      CPUFREQ_START, policy);
1250
1251         if (new_policy) {
1252                 ret = cpufreq_add_dev_interface(policy);
1253                 if (ret)
1254                         goto out_exit_policy;
1255                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1256                                 CPUFREQ_CREATE_POLICY, policy);
1257
1258                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1259                 list_add(&policy->policy_list, &cpufreq_policy_list);
1260                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1261         }
1262
1263         ret = cpufreq_init_policy(policy);
1264         if (ret) {
1265                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1266                        __func__, cpu, ret);
1267                 /* cpufreq_policy_free() will notify based on this */
1268                 new_policy = false;
1269                 goto out_exit_policy;
1270         }
1271
1272         up_write(&policy->rwsem);
1273
1274         kobject_uevent(&policy->kobj, KOBJ_ADD);
1275
1276         /* Callback for handling stuff after policy is ready */
1277         if (cpufreq_driver->ready)
1278                 cpufreq_driver->ready(policy);
1279
1280         pr_debug("initialization complete\n");
1281
1282         return 0;
1283
1284 out_exit_policy:
1285         up_write(&policy->rwsem);
1286
1287         if (cpufreq_driver->exit)
1288                 cpufreq_driver->exit(policy);
1289 out_free_policy:
1290         cpufreq_policy_free(policy, !new_policy);
1291         return ret;
1292 }
1293
1294 /**
1295  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1296  * @dev: CPU device.
1297  * @sif: Subsystem interface structure pointer (not used)
1298  */
1299 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1300 {
1301         unsigned cpu = dev->id;
1302         int ret;
1303
1304         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1305
1306         if (cpu_online(cpu)) {
1307                 ret = cpufreq_online(cpu);
1308         } else {
1309                 /*
1310                  * A hotplug notifier will follow and we will handle it as CPU
1311                  * online then.  For now, just create the sysfs link, unless
1312                  * there is no policy or the link is already present.
1313                  */
1314                 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1315
1316                 ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
1317                         ? add_cpu_dev_symlink(policy, cpu) : 0;
1318         }
1319
1320         return ret;
1321 }
1322
1323 static void cpufreq_offline_prepare(unsigned int cpu)
1324 {
1325         struct cpufreq_policy *policy;
1326
1327         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1328
1329         policy = cpufreq_cpu_get_raw(cpu);
1330         if (!policy) {
1331                 pr_debug("%s: No cpu_data found\n", __func__);
1332                 return;
1333         }
1334
1335         if (has_target()) {
1336                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1337                 if (ret)
1338                         pr_err("%s: Failed to stop governor\n", __func__);
1339         }
1340
1341         down_write(&policy->rwsem);
1342         cpumask_clear_cpu(cpu, policy->cpus);
1343
1344         if (policy_is_inactive(policy)) {
1345                 if (has_target())
1346                         strncpy(policy->last_governor, policy->governor->name,
1347                                 CPUFREQ_NAME_LEN);
1348                 else
1349                         policy->last_policy = policy->policy;
1350         } else if (cpu == policy->cpu) {
1351                 /* Nominate new CPU */
1352                 policy->cpu = cpumask_any(policy->cpus);
1353         }
1354         up_write(&policy->rwsem);
1355
1356         /* Start governor again for active policy */
1357         if (!policy_is_inactive(policy)) {
1358                 if (has_target()) {
1359                         int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1360                         if (!ret)
1361                                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1362
1363                         if (ret)
1364                                 pr_err("%s: Failed to start governor\n", __func__);
1365                 }
1366         } else if (cpufreq_driver->stop_cpu) {
1367                 cpufreq_driver->stop_cpu(policy);
1368         }
1369 }
1370
1371 static void cpufreq_offline_finish(unsigned int cpu)
1372 {
1373         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1374
1375         if (!policy) {
1376                 pr_debug("%s: No cpu_data found\n", __func__);
1377                 return;
1378         }
1379
1380         /* Only proceed for inactive policies */
1381         if (!policy_is_inactive(policy))
1382                 return;
1383
1384         /* If cpu is last user of policy, free policy */
1385         if (has_target()) {
1386                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1387                 if (ret)
1388                         pr_err("%s: Failed to exit governor\n", __func__);
1389         }
1390
1391         /*
1392          * Perform the ->exit() even during light-weight tear-down,
1393          * since this is a core component, and is essential for the
1394          * subsequent light-weight ->init() to succeed.
1395          */
1396         if (cpufreq_driver->exit) {
1397                 cpufreq_driver->exit(policy);
1398                 policy->freq_table = NULL;
1399         }
1400 }
1401
1402 /**
1403  * cpufreq_remove_dev - remove a CPU device
1404  *
1405  * Removes the cpufreq interface for a CPU device.
1406  */
1407 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1408 {
1409         unsigned int cpu = dev->id;
1410         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1411
1412         if (!policy)
1413                 return;
1414
1415         if (cpu_online(cpu)) {
1416                 cpufreq_offline_prepare(cpu);
1417                 cpufreq_offline_finish(cpu);
1418         }
1419
1420         cpumask_clear_cpu(cpu, policy->real_cpus);
1421         remove_cpu_dev_symlink(policy, cpu);
1422
1423         if (cpumask_empty(policy->real_cpus))
1424                 cpufreq_policy_free(policy, true);
1425 }
1426
1427 static void handle_update(struct work_struct *work)
1428 {
1429         struct cpufreq_policy *policy =
1430                 container_of(work, struct cpufreq_policy, update);
1431         unsigned int cpu = policy->cpu;
1432         pr_debug("handle_update for cpu %u called\n", cpu);
1433         cpufreq_update_policy(cpu);
1434 }
1435
1436 /**
1437  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1438  *      in deep trouble.
1439  *      @policy: policy managing CPUs
1440  *      @new_freq: CPU frequency the CPU actually runs at
1441  *
1442  *      We adjust to current frequency first, and need to clean up later.
1443  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1444  */
1445 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1446                                 unsigned int new_freq)
1447 {
1448         struct cpufreq_freqs freqs;
1449
1450         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1451                  policy->cur, new_freq);
1452
1453         freqs.old = policy->cur;
1454         freqs.new = new_freq;
1455
1456         cpufreq_freq_transition_begin(policy, &freqs);
1457         cpufreq_freq_transition_end(policy, &freqs, 0);
1458 }
1459
1460 /**
1461  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1462  * @cpu: CPU number
1463  *
1464  * This is the last known freq, without actually getting it from the driver.
1465  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1466  */
1467 unsigned int cpufreq_quick_get(unsigned int cpu)
1468 {
1469         struct cpufreq_policy *policy;
1470         unsigned int ret_freq = 0;
1471
1472         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1473                 return cpufreq_driver->get(cpu);
1474
1475         policy = cpufreq_cpu_get(cpu);
1476         if (policy) {
1477                 ret_freq = policy->cur;
1478                 cpufreq_cpu_put(policy);
1479         }
1480
1481         return ret_freq;
1482 }
1483 EXPORT_SYMBOL(cpufreq_quick_get);
1484
1485 /**
1486  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1487  * @cpu: CPU number
1488  *
1489  * Just return the max possible frequency for a given CPU.
1490  */
1491 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1492 {
1493         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1494         unsigned int ret_freq = 0;
1495
1496         if (policy) {
1497                 ret_freq = policy->max;
1498                 cpufreq_cpu_put(policy);
1499         }
1500
1501         return ret_freq;
1502 }
1503 EXPORT_SYMBOL(cpufreq_quick_get_max);
1504
1505 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1506 {
1507         unsigned int ret_freq = 0;
1508
1509         if (!cpufreq_driver->get)
1510                 return ret_freq;
1511
1512         ret_freq = cpufreq_driver->get(policy->cpu);
1513
1514         /* Updating inactive policies is invalid, so avoid doing that. */
1515         if (unlikely(policy_is_inactive(policy)))
1516                 return ret_freq;
1517
1518         if (ret_freq && policy->cur &&
1519                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1520                 /* verify no discrepancy between actual and
1521                                         saved value exists */
1522                 if (unlikely(ret_freq != policy->cur)) {
1523                         cpufreq_out_of_sync(policy, ret_freq);
1524                         schedule_work(&policy->update);
1525                 }
1526         }
1527
1528         return ret_freq;
1529 }
1530
1531 /**
1532  * cpufreq_get - get the current CPU frequency (in kHz)
1533  * @cpu: CPU number
1534  *
1535  * Get the CPU current (static) CPU frequency
1536  */
1537 unsigned int cpufreq_get(unsigned int cpu)
1538 {
1539         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1540         unsigned int ret_freq = 0;
1541
1542         if (policy) {
1543                 down_read(&policy->rwsem);
1544                 ret_freq = __cpufreq_get(policy);
1545                 up_read(&policy->rwsem);
1546
1547                 cpufreq_cpu_put(policy);
1548         }
1549
1550         return ret_freq;
1551 }
1552 EXPORT_SYMBOL(cpufreq_get);
1553
1554 static struct subsys_interface cpufreq_interface = {
1555         .name           = "cpufreq",
1556         .subsys         = &cpu_subsys,
1557         .add_dev        = cpufreq_add_dev,
1558         .remove_dev     = cpufreq_remove_dev,
1559 };
1560
1561 /*
1562  * In case platform wants some specific frequency to be configured
1563  * during suspend..
1564  */
1565 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1566 {
1567         int ret;
1568
1569         if (!policy->suspend_freq) {
1570                 pr_debug("%s: suspend_freq not defined\n", __func__);
1571                 return 0;
1572         }
1573
1574         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1575                         policy->suspend_freq);
1576
1577         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1578                         CPUFREQ_RELATION_H);
1579         if (ret)
1580                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1581                                 __func__, policy->suspend_freq, ret);
1582
1583         return ret;
1584 }
1585 EXPORT_SYMBOL(cpufreq_generic_suspend);
1586
1587 /**
1588  * cpufreq_suspend() - Suspend CPUFreq governors
1589  *
1590  * Called during system wide Suspend/Hibernate cycles for suspending governors
1591  * as some platforms can't change frequency after this point in suspend cycle.
1592  * Because some of the devices (like: i2c, regulators, etc) they use for
1593  * changing frequency are suspended quickly after this point.
1594  */
1595 void cpufreq_suspend(void)
1596 {
1597         struct cpufreq_policy *policy;
1598
1599         if (!cpufreq_driver)
1600                 return;
1601
1602         if (!has_target())
1603                 goto suspend;
1604
1605         pr_debug("%s: Suspending Governors\n", __func__);
1606
1607         for_each_active_policy(policy) {
1608                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1609                         pr_err("%s: Failed to stop governor for policy: %p\n",
1610                                 __func__, policy);
1611                 else if (cpufreq_driver->suspend
1612                     && cpufreq_driver->suspend(policy))
1613                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1614                                 policy);
1615         }
1616
1617 suspend:
1618         cpufreq_suspended = true;
1619 }
1620
1621 /**
1622  * cpufreq_resume() - Resume CPUFreq governors
1623  *
1624  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1625  * are suspended with cpufreq_suspend().
1626  */
1627 void cpufreq_resume(void)
1628 {
1629         struct cpufreq_policy *policy;
1630
1631         if (!cpufreq_driver)
1632                 return;
1633
1634         cpufreq_suspended = false;
1635
1636         if (!has_target())
1637                 return;
1638
1639         pr_debug("%s: Resuming Governors\n", __func__);
1640
1641         for_each_active_policy(policy) {
1642                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1643                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1644                                 policy);
1645                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1646                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1647                         pr_err("%s: Failed to start governor for policy: %p\n",
1648                                 __func__, policy);
1649         }
1650
1651         /*
1652          * schedule call cpufreq_update_policy() for first-online CPU, as that
1653          * wouldn't be hotplugged-out on suspend. It will verify that the
1654          * current freq is in sync with what we believe it to be.
1655          */
1656         policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
1657         if (WARN_ON(!policy))
1658                 return;
1659
1660         schedule_work(&policy->update);
1661 }
1662
1663 /**
1664  *      cpufreq_get_current_driver - return current driver's name
1665  *
1666  *      Return the name string of the currently loaded cpufreq driver
1667  *      or NULL, if none.
1668  */
1669 const char *cpufreq_get_current_driver(void)
1670 {
1671         if (cpufreq_driver)
1672                 return cpufreq_driver->name;
1673
1674         return NULL;
1675 }
1676 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1677
1678 /**
1679  *      cpufreq_get_driver_data - return current driver data
1680  *
1681  *      Return the private data of the currently loaded cpufreq
1682  *      driver, or NULL if no cpufreq driver is loaded.
1683  */
1684 void *cpufreq_get_driver_data(void)
1685 {
1686         if (cpufreq_driver)
1687                 return cpufreq_driver->driver_data;
1688
1689         return NULL;
1690 }
1691 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1692
1693 /*********************************************************************
1694  *                     NOTIFIER LISTS INTERFACE                      *
1695  *********************************************************************/
1696
1697 /**
1698  *      cpufreq_register_notifier - register a driver with cpufreq
1699  *      @nb: notifier function to register
1700  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1701  *
1702  *      Add a driver to one of two lists: either a list of drivers that
1703  *      are notified about clock rate changes (once before and once after
1704  *      the transition), or a list of drivers that are notified about
1705  *      changes in cpufreq policy.
1706  *
1707  *      This function may sleep, and has the same return conditions as
1708  *      blocking_notifier_chain_register.
1709  */
1710 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1711 {
1712         int ret;
1713
1714         if (cpufreq_disabled())
1715                 return -EINVAL;
1716
1717         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1718
1719         switch (list) {
1720         case CPUFREQ_TRANSITION_NOTIFIER:
1721                 ret = srcu_notifier_chain_register(
1722                                 &cpufreq_transition_notifier_list, nb);
1723                 break;
1724         case CPUFREQ_POLICY_NOTIFIER:
1725                 ret = blocking_notifier_chain_register(
1726                                 &cpufreq_policy_notifier_list, nb);
1727                 break;
1728         default:
1729                 ret = -EINVAL;
1730         }
1731
1732         return ret;
1733 }
1734 EXPORT_SYMBOL(cpufreq_register_notifier);
1735
1736 /**
1737  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1738  *      @nb: notifier block to be unregistered
1739  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1740  *
1741  *      Remove a driver from the CPU frequency notifier list.
1742  *
1743  *      This function may sleep, and has the same return conditions as
1744  *      blocking_notifier_chain_unregister.
1745  */
1746 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1747 {
1748         int ret;
1749
1750         if (cpufreq_disabled())
1751                 return -EINVAL;
1752
1753         switch (list) {
1754         case CPUFREQ_TRANSITION_NOTIFIER:
1755                 ret = srcu_notifier_chain_unregister(
1756                                 &cpufreq_transition_notifier_list, nb);
1757                 break;
1758         case CPUFREQ_POLICY_NOTIFIER:
1759                 ret = blocking_notifier_chain_unregister(
1760                                 &cpufreq_policy_notifier_list, nb);
1761                 break;
1762         default:
1763                 ret = -EINVAL;
1764         }
1765
1766         return ret;
1767 }
1768 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1769
1770
1771 /*********************************************************************
1772  *                              GOVERNORS                            *
1773  *********************************************************************/
1774
1775 /* Must set freqs->new to intermediate frequency */
1776 static int __target_intermediate(struct cpufreq_policy *policy,
1777                                  struct cpufreq_freqs *freqs, int index)
1778 {
1779         int ret;
1780
1781         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1782
1783         /* We don't need to switch to intermediate freq */
1784         if (!freqs->new)
1785                 return 0;
1786
1787         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1788                  __func__, policy->cpu, freqs->old, freqs->new);
1789
1790         cpufreq_freq_transition_begin(policy, freqs);
1791         ret = cpufreq_driver->target_intermediate(policy, index);
1792         cpufreq_freq_transition_end(policy, freqs, ret);
1793
1794         if (ret)
1795                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1796                        __func__, ret);
1797
1798         return ret;
1799 }
1800
1801 static int __target_index(struct cpufreq_policy *policy,
1802                           struct cpufreq_frequency_table *freq_table, int index)
1803 {
1804         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1805         unsigned int intermediate_freq = 0;
1806         int retval = -EINVAL;
1807         bool notify;
1808
1809         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1810         if (notify) {
1811                 /* Handle switching to intermediate frequency */
1812                 if (cpufreq_driver->get_intermediate) {
1813                         retval = __target_intermediate(policy, &freqs, index);
1814                         if (retval)
1815                                 return retval;
1816
1817                         intermediate_freq = freqs.new;
1818                         /* Set old freq to intermediate */
1819                         if (intermediate_freq)
1820                                 freqs.old = freqs.new;
1821                 }
1822
1823                 freqs.new = freq_table[index].frequency;
1824                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1825                          __func__, policy->cpu, freqs.old, freqs.new);
1826
1827                 cpufreq_freq_transition_begin(policy, &freqs);
1828         }
1829
1830         retval = cpufreq_driver->target_index(policy, index);
1831         if (retval)
1832                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1833                        retval);
1834
1835         if (notify) {
1836                 cpufreq_freq_transition_end(policy, &freqs, retval);
1837
1838                 /*
1839                  * Failed after setting to intermediate freq? Driver should have
1840                  * reverted back to initial frequency and so should we. Check
1841                  * here for intermediate_freq instead of get_intermediate, in
1842                  * case we haven't switched to intermediate freq at all.
1843                  */
1844                 if (unlikely(retval && intermediate_freq)) {
1845                         freqs.old = intermediate_freq;
1846                         freqs.new = policy->restore_freq;
1847                         cpufreq_freq_transition_begin(policy, &freqs);
1848                         cpufreq_freq_transition_end(policy, &freqs, 0);
1849                 }
1850         }
1851
1852         return retval;
1853 }
1854
1855 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1856                             unsigned int target_freq,
1857                             unsigned int relation)
1858 {
1859         unsigned int old_target_freq = target_freq;
1860         int retval = -EINVAL;
1861
1862         if (cpufreq_disabled())
1863                 return -ENODEV;
1864
1865         /* Make sure that target_freq is within supported range */
1866         if (target_freq > policy->max)
1867                 target_freq = policy->max;
1868         if (target_freq < policy->min)
1869                 target_freq = policy->min;
1870
1871         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1872                  policy->cpu, target_freq, relation, old_target_freq);
1873
1874         /*
1875          * This might look like a redundant call as we are checking it again
1876          * after finding index. But it is left intentionally for cases where
1877          * exactly same freq is called again and so we can save on few function
1878          * calls.
1879          */
1880         if (target_freq == policy->cur)
1881                 return 0;
1882
1883         /* Save last value to restore later on errors */
1884         policy->restore_freq = policy->cur;
1885
1886         if (cpufreq_driver->target)
1887                 retval = cpufreq_driver->target(policy, target_freq, relation);
1888         else if (cpufreq_driver->target_index) {
1889                 struct cpufreq_frequency_table *freq_table;
1890                 int index;
1891
1892                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1893                 if (unlikely(!freq_table)) {
1894                         pr_err("%s: Unable to find freq_table\n", __func__);
1895                         goto out;
1896                 }
1897
1898                 retval = cpufreq_frequency_table_target(policy, freq_table,
1899                                 target_freq, relation, &index);
1900                 if (unlikely(retval)) {
1901                         pr_err("%s: Unable to find matching freq\n", __func__);
1902                         goto out;
1903                 }
1904
1905                 if (freq_table[index].frequency == policy->cur) {
1906                         retval = 0;
1907                         goto out;
1908                 }
1909
1910                 retval = __target_index(policy, freq_table, index);
1911         }
1912
1913 out:
1914         return retval;
1915 }
1916 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1917
1918 int cpufreq_driver_target(struct cpufreq_policy *policy,
1919                           unsigned int target_freq,
1920                           unsigned int relation)
1921 {
1922         int ret = -EINVAL;
1923
1924         down_write(&policy->rwsem);
1925
1926         ret = __cpufreq_driver_target(policy, target_freq, relation);
1927
1928         up_write(&policy->rwsem);
1929
1930         return ret;
1931 }
1932 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1933
1934 static int __cpufreq_governor(struct cpufreq_policy *policy,
1935                                         unsigned int event)
1936 {
1937         int ret;
1938
1939         /* Only must be defined when default governor is known to have latency
1940            restrictions, like e.g. conservative or ondemand.
1941            That this is the case is already ensured in Kconfig
1942         */
1943 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1944         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1945 #else
1946         struct cpufreq_governor *gov = NULL;
1947 #endif
1948
1949         /* Don't start any governor operations if we are entering suspend */
1950         if (cpufreq_suspended)
1951                 return 0;
1952         /*
1953          * Governor might not be initiated here if ACPI _PPC changed
1954          * notification happened, so check it.
1955          */
1956         if (!policy->governor)
1957                 return -EINVAL;
1958
1959         if (policy->governor->max_transition_latency &&
1960             policy->cpuinfo.transition_latency >
1961             policy->governor->max_transition_latency) {
1962                 if (!gov)
1963                         return -EINVAL;
1964                 else {
1965                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
1966                                 policy->governor->name, gov->name);
1967                         policy->governor = gov;
1968                 }
1969         }
1970
1971         if (event == CPUFREQ_GOV_POLICY_INIT)
1972                 if (!try_module_get(policy->governor->owner))
1973                         return -EINVAL;
1974
1975         pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
1976
1977         mutex_lock(&cpufreq_governor_lock);
1978         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1979             || (!policy->governor_enabled
1980             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1981                 mutex_unlock(&cpufreq_governor_lock);
1982                 return -EBUSY;
1983         }
1984
1985         if (event == CPUFREQ_GOV_STOP)
1986                 policy->governor_enabled = false;
1987         else if (event == CPUFREQ_GOV_START)
1988                 policy->governor_enabled = true;
1989
1990         mutex_unlock(&cpufreq_governor_lock);
1991
1992         ret = policy->governor->governor(policy, event);
1993
1994         if (!ret) {
1995                 if (event == CPUFREQ_GOV_POLICY_INIT)
1996                         policy->governor->initialized++;
1997                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1998                         policy->governor->initialized--;
1999         } else {
2000                 /* Restore original values */
2001                 mutex_lock(&cpufreq_governor_lock);
2002                 if (event == CPUFREQ_GOV_STOP)
2003                         policy->governor_enabled = true;
2004                 else if (event == CPUFREQ_GOV_START)
2005                         policy->governor_enabled = false;
2006                 mutex_unlock(&cpufreq_governor_lock);
2007         }
2008
2009         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2010                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2011                 module_put(policy->governor->owner);
2012
2013         return ret;
2014 }
2015
2016 int cpufreq_register_governor(struct cpufreq_governor *governor)
2017 {
2018         int err;
2019
2020         if (!governor)
2021                 return -EINVAL;
2022
2023         if (cpufreq_disabled())
2024                 return -ENODEV;
2025
2026         mutex_lock(&cpufreq_governor_mutex);
2027
2028         governor->initialized = 0;
2029         err = -EBUSY;
2030         if (!find_governor(governor->name)) {
2031                 err = 0;
2032                 list_add(&governor->governor_list, &cpufreq_governor_list);
2033         }
2034
2035         mutex_unlock(&cpufreq_governor_mutex);
2036         return err;
2037 }
2038 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2039
2040 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2041 {
2042         struct cpufreq_policy *policy;
2043         unsigned long flags;
2044
2045         if (!governor)
2046                 return;
2047
2048         if (cpufreq_disabled())
2049                 return;
2050
2051         /* clear last_governor for all inactive policies */
2052         read_lock_irqsave(&cpufreq_driver_lock, flags);
2053         for_each_inactive_policy(policy) {
2054                 if (!strcmp(policy->last_governor, governor->name)) {
2055                         policy->governor = NULL;
2056                         strcpy(policy->last_governor, "\0");
2057                 }
2058         }
2059         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2060
2061         mutex_lock(&cpufreq_governor_mutex);
2062         list_del(&governor->governor_list);
2063         mutex_unlock(&cpufreq_governor_mutex);
2064         return;
2065 }
2066 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2067
2068
2069 /*********************************************************************
2070  *                          POLICY INTERFACE                         *
2071  *********************************************************************/
2072
2073 /**
2074  * cpufreq_get_policy - get the current cpufreq_policy
2075  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2076  *      is written
2077  *
2078  * Reads the current cpufreq policy.
2079  */
2080 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2081 {
2082         struct cpufreq_policy *cpu_policy;
2083         if (!policy)
2084                 return -EINVAL;
2085
2086         cpu_policy = cpufreq_cpu_get(cpu);
2087         if (!cpu_policy)
2088                 return -EINVAL;
2089
2090         memcpy(policy, cpu_policy, sizeof(*policy));
2091
2092         cpufreq_cpu_put(cpu_policy);
2093         return 0;
2094 }
2095 EXPORT_SYMBOL(cpufreq_get_policy);
2096
2097 /*
2098  * policy : current policy.
2099  * new_policy: policy to be set.
2100  */
2101 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2102                                 struct cpufreq_policy *new_policy)
2103 {
2104         struct cpufreq_governor *old_gov;
2105         int ret;
2106
2107         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2108                  new_policy->cpu, new_policy->min, new_policy->max);
2109
2110         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2111
2112         /*
2113         * This check works well when we store new min/max freq attributes,
2114         * because new_policy is a copy of policy with one field updated.
2115         */
2116         if (new_policy->min > new_policy->max)
2117                 return -EINVAL;
2118
2119         /* verify the cpu speed can be set within this limit */
2120         ret = cpufreq_driver->verify(new_policy);
2121         if (ret)
2122                 return ret;
2123
2124         /* adjust if necessary - all reasons */
2125         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2126                         CPUFREQ_ADJUST, new_policy);
2127
2128         /*
2129          * verify the cpu speed can be set within this limit, which might be
2130          * different to the first one
2131          */
2132         ret = cpufreq_driver->verify(new_policy);
2133         if (ret)
2134                 return ret;
2135
2136         /* notification of the new policy */
2137         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2138                         CPUFREQ_NOTIFY, new_policy);
2139
2140         policy->min = new_policy->min;
2141         policy->max = new_policy->max;
2142
2143         pr_debug("new min and max freqs are %u - %u kHz\n",
2144                  policy->min, policy->max);
2145
2146         if (cpufreq_driver->setpolicy) {
2147                 policy->policy = new_policy->policy;
2148                 pr_debug("setting range\n");
2149                 return cpufreq_driver->setpolicy(new_policy);
2150         }
2151
2152         if (new_policy->governor == policy->governor)
2153                 goto out;
2154
2155         pr_debug("governor switch\n");
2156
2157         /* save old, working values */
2158         old_gov = policy->governor;
2159         /* end old governor */
2160         if (old_gov) {
2161                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2162                 if (ret) {
2163                         /* This can happen due to race with other operations */
2164                         pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
2165                                  __func__, old_gov->name, ret);
2166                         return ret;
2167                 }
2168
2169                 up_write(&policy->rwsem);
2170                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2171                 down_write(&policy->rwsem);
2172
2173                 if (ret) {
2174                         pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2175                                __func__, old_gov->name, ret);
2176                         return ret;
2177                 }
2178         }
2179
2180         /* start new governor */
2181         policy->governor = new_policy->governor;
2182         ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2183         if (!ret) {
2184                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
2185                 if (!ret)
2186                         goto out;
2187
2188                 up_write(&policy->rwsem);
2189                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2190                 down_write(&policy->rwsem);
2191         }
2192
2193         /* new governor failed, so re-start old one */
2194         pr_debug("starting governor %s failed\n", policy->governor->name);
2195         if (old_gov) {
2196                 policy->governor = old_gov;
2197                 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2198                         policy->governor = NULL;
2199                 else
2200                         __cpufreq_governor(policy, CPUFREQ_GOV_START);
2201         }
2202
2203         return ret;
2204
2205  out:
2206         pr_debug("governor: change or update limits\n");
2207         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2208 }
2209
2210 /**
2211  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2212  *      @cpu: CPU which shall be re-evaluated
2213  *
2214  *      Useful for policy notifiers which have different necessities
2215  *      at different times.
2216  */
2217 int cpufreq_update_policy(unsigned int cpu)
2218 {
2219         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2220         struct cpufreq_policy new_policy;
2221         int ret;
2222
2223         if (!policy)
2224                 return -ENODEV;
2225
2226         down_write(&policy->rwsem);
2227
2228         pr_debug("updating policy for CPU %u\n", cpu);
2229         memcpy(&new_policy, policy, sizeof(*policy));
2230         new_policy.min = policy->user_policy.min;
2231         new_policy.max = policy->user_policy.max;
2232
2233         /*
2234          * BIOS might change freq behind our back
2235          * -> ask driver for current freq and notify governors about a change
2236          */
2237         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2238                 new_policy.cur = cpufreq_driver->get(cpu);
2239                 if (WARN_ON(!new_policy.cur)) {
2240                         ret = -EIO;
2241                         goto unlock;
2242                 }
2243
2244                 if (!policy->cur) {
2245                         pr_debug("Driver did not initialize current freq\n");
2246                         policy->cur = new_policy.cur;
2247                 } else {
2248                         if (policy->cur != new_policy.cur && has_target())
2249                                 cpufreq_out_of_sync(policy, new_policy.cur);
2250                 }
2251         }
2252
2253         ret = cpufreq_set_policy(policy, &new_policy);
2254
2255 unlock:
2256         up_write(&policy->rwsem);
2257
2258         cpufreq_cpu_put(policy);
2259         return ret;
2260 }
2261 EXPORT_SYMBOL(cpufreq_update_policy);
2262
2263 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2264                                         unsigned long action, void *hcpu)
2265 {
2266         unsigned int cpu = (unsigned long)hcpu;
2267
2268         switch (action & ~CPU_TASKS_FROZEN) {
2269         case CPU_ONLINE:
2270                 cpufreq_online(cpu);
2271                 break;
2272
2273         case CPU_DOWN_PREPARE:
2274                 cpufreq_offline_prepare(cpu);
2275                 break;
2276
2277         case CPU_POST_DEAD:
2278                 cpufreq_offline_finish(cpu);
2279                 break;
2280
2281         case CPU_DOWN_FAILED:
2282                 cpufreq_online(cpu);
2283                 break;
2284         }
2285         return NOTIFY_OK;
2286 }
2287
2288 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2289         .notifier_call = cpufreq_cpu_callback,
2290 };
2291
2292 /*********************************************************************
2293  *               BOOST                                               *
2294  *********************************************************************/
2295 static int cpufreq_boost_set_sw(int state)
2296 {
2297         struct cpufreq_frequency_table *freq_table;
2298         struct cpufreq_policy *policy;
2299         int ret = -EINVAL;
2300
2301         for_each_active_policy(policy) {
2302                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2303                 if (freq_table) {
2304                         ret = cpufreq_frequency_table_cpuinfo(policy,
2305                                                         freq_table);
2306                         if (ret) {
2307                                 pr_err("%s: Policy frequency update failed\n",
2308                                        __func__);
2309                                 break;
2310                         }
2311                         policy->user_policy.max = policy->max;
2312                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2313                 }
2314         }
2315
2316         return ret;
2317 }
2318
2319 int cpufreq_boost_trigger_state(int state)
2320 {
2321         unsigned long flags;
2322         int ret = 0;
2323
2324         if (cpufreq_driver->boost_enabled == state)
2325                 return 0;
2326
2327         write_lock_irqsave(&cpufreq_driver_lock, flags);
2328         cpufreq_driver->boost_enabled = state;
2329         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2330
2331         ret = cpufreq_driver->set_boost(state);
2332         if (ret) {
2333                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2334                 cpufreq_driver->boost_enabled = !state;
2335                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2336
2337                 pr_err("%s: Cannot %s BOOST\n",
2338                        __func__, state ? "enable" : "disable");
2339         }
2340
2341         return ret;
2342 }
2343
2344 int cpufreq_boost_supported(void)
2345 {
2346         if (likely(cpufreq_driver))
2347                 return cpufreq_driver->boost_supported;
2348
2349         return 0;
2350 }
2351 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2352
2353 static int create_boost_sysfs_file(void)
2354 {
2355         int ret;
2356
2357         if (!cpufreq_boost_supported())
2358                 return 0;
2359
2360         /*
2361          * Check if driver provides function to enable boost -
2362          * if not, use cpufreq_boost_set_sw as default
2363          */
2364         if (!cpufreq_driver->set_boost)
2365                 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2366
2367         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2368         if (ret)
2369                 pr_err("%s: cannot register global BOOST sysfs file\n",
2370                        __func__);
2371
2372         return ret;
2373 }
2374
2375 static void remove_boost_sysfs_file(void)
2376 {
2377         if (cpufreq_boost_supported())
2378                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2379 }
2380
2381 int cpufreq_enable_boost_support(void)
2382 {
2383         if (!cpufreq_driver)
2384                 return -EINVAL;
2385
2386         if (cpufreq_boost_supported())
2387                 return 0;
2388
2389         cpufreq_driver->boost_supported = true;
2390
2391         /* This will get removed on driver unregister */
2392         return create_boost_sysfs_file();
2393 }
2394 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2395
2396 int cpufreq_boost_enabled(void)
2397 {
2398         return cpufreq_driver->boost_enabled;
2399 }
2400 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2401
2402 /*********************************************************************
2403  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2404  *********************************************************************/
2405
2406 /**
2407  * cpufreq_register_driver - register a CPU Frequency driver
2408  * @driver_data: A struct cpufreq_driver containing the values#
2409  * submitted by the CPU Frequency driver.
2410  *
2411  * Registers a CPU Frequency driver to this core code. This code
2412  * returns zero on success, -EBUSY when another driver got here first
2413  * (and isn't unregistered in the meantime).
2414  *
2415  */
2416 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2417 {
2418         unsigned long flags;
2419         int ret;
2420
2421         if (cpufreq_disabled())
2422                 return -ENODEV;
2423
2424         if (!driver_data || !driver_data->verify || !driver_data->init ||
2425             !(driver_data->setpolicy || driver_data->target_index ||
2426                     driver_data->target) ||
2427              (driver_data->setpolicy && (driver_data->target_index ||
2428                     driver_data->target)) ||
2429              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2430                 return -EINVAL;
2431
2432         pr_debug("trying to register driver %s\n", driver_data->name);
2433
2434         /* Protect against concurrent CPU online/offline. */
2435         get_online_cpus();
2436
2437         write_lock_irqsave(&cpufreq_driver_lock, flags);
2438         if (cpufreq_driver) {
2439                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2440                 ret = -EEXIST;
2441                 goto out;
2442         }
2443         cpufreq_driver = driver_data;
2444         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2445
2446         if (driver_data->setpolicy)
2447                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2448
2449         ret = create_boost_sysfs_file();
2450         if (ret)
2451                 goto err_null_driver;
2452
2453         ret = subsys_interface_register(&cpufreq_interface);
2454         if (ret)
2455                 goto err_boost_unreg;
2456
2457         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2458             list_empty(&cpufreq_policy_list)) {
2459                 /* if all ->init() calls failed, unregister */
2460                 ret = -ENODEV;
2461                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2462                          driver_data->name);
2463                 goto err_if_unreg;
2464         }
2465
2466         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2467         pr_debug("driver %s up and running\n", driver_data->name);
2468
2469 out:
2470         put_online_cpus();
2471         return ret;
2472
2473 err_if_unreg:
2474         subsys_interface_unregister(&cpufreq_interface);
2475 err_boost_unreg:
2476         remove_boost_sysfs_file();
2477 err_null_driver:
2478         write_lock_irqsave(&cpufreq_driver_lock, flags);
2479         cpufreq_driver = NULL;
2480         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2481         goto out;
2482 }
2483 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2484
2485 /**
2486  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2487  *
2488  * Unregister the current CPUFreq driver. Only call this if you have
2489  * the right to do so, i.e. if you have succeeded in initialising before!
2490  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2491  * currently not initialised.
2492  */
2493 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2494 {
2495         unsigned long flags;
2496
2497         if (!cpufreq_driver || (driver != cpufreq_driver))
2498                 return -EINVAL;
2499
2500         pr_debug("unregistering driver %s\n", driver->name);
2501
2502         /* Protect against concurrent cpu hotplug */
2503         get_online_cpus();
2504         subsys_interface_unregister(&cpufreq_interface);
2505         remove_boost_sysfs_file();
2506         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2507
2508         write_lock_irqsave(&cpufreq_driver_lock, flags);
2509
2510         cpufreq_driver = NULL;
2511
2512         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2513         put_online_cpus();
2514
2515         return 0;
2516 }
2517 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2518
2519 /*
2520  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2521  * or mutexes when secondary CPUs are halted.
2522  */
2523 static struct syscore_ops cpufreq_syscore_ops = {
2524         .shutdown = cpufreq_suspend,
2525 };
2526
2527 struct kobject *cpufreq_global_kobject;
2528 EXPORT_SYMBOL(cpufreq_global_kobject);
2529
2530 static int __init cpufreq_core_init(void)
2531 {
2532         if (cpufreq_disabled())
2533                 return -ENODEV;
2534
2535         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2536         BUG_ON(!cpufreq_global_kobject);
2537
2538         register_syscore_ops(&cpufreq_syscore_ops);
2539
2540         return 0;
2541 }
2542 core_initcall(cpufreq_core_init);