drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #ifdef CONFIG_SMP
  33 #include <linux/sched.h>
  34 #endif
  35 #include <trace/events/power.h>
  36
  37 static LIST_HEAD(cpufreq_policy_list);
  38
  39 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  40 {
  41         return cpumask_empty(policy->cpus);
  42 }
  43
  44 static bool suitable_policy(struct cpufreq_policy *policy, bool active)
  45 {
  46         return active == !policy_is_inactive(policy);
  47 }
  48
  49 /* Finds Next Acive/Inactive policy */
  50 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
  51                                           bool active)
  52 {
  53         do {
  54                 policy = list_next_entry(policy, policy_list);
  55
  56                 /* No more policies in the list */
  57                 if (&policy->policy_list == &cpufreq_policy_list)
  58                         return NULL;
  59         } while (!suitable_policy(policy, active));
  60
  61         return policy;
  62 }
  63
  64 static struct cpufreq_policy *first_policy(bool active)
  65 {
  66         struct cpufreq_policy *policy;
  67
  68         /* No policies in the list */
  69         if (list_empty(&cpufreq_policy_list))
  70                 return NULL;
  71
  72         policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
  73                                   policy_list);
  74
  75         if (!suitable_policy(policy, active))
  76                 policy = next_policy(policy, active);
  77
  78         return policy;
  79 }
  80
  81 /* Macros to iterate over CPU policies */
  82 #define for_each_suitable_policy(__policy, __active)    \
  83         for (__policy = first_policy(__active);         \
  84              __policy;                                  \
  85              __policy = next_policy(__policy, __active))
  86
  87 #define for_each_active_policy(__policy)                \
  88         for_each_suitable_policy(__policy, true)
  89 #define for_each_inactive_policy(__policy)              \
  90         for_each_suitable_policy(__policy, false)
  91
  92 #define for_each_policy(__policy)                       \
  93         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  94
  95 /* Iterate over governors */
  96 static LIST_HEAD(cpufreq_governor_list);
  97 #define for_each_governor(__governor)                           \
  98         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  99
 100 /**
 101  * The "cpufreq driver" - the arch- or hardware-dependent low
 102  * level driver of CPUFreq support, and its spinlock. This lock
 103  * also protects the cpufreq_cpu_data array.
 104  */
 105 static struct cpufreq_driver *cpufreq_driver;
 106 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
 107 static DEFINE_RWLOCK(cpufreq_driver_lock);
 108 DEFINE_MUTEX(cpufreq_governor_lock);
 109
 110 /* Flag to suspend/resume CPUFreq governors */
 111 static bool cpufreq_suspended;
 112
 113 static inline bool has_target(void)
 114 {
 115         return cpufreq_driver->target_index || cpufreq_driver->target;
 116 }
 117
 118 /* internal prototypes */
 119 static int __cpufreq_governor(struct cpufreq_policy *policy,
 120                 unsigned int event);
 121 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 122 static void handle_update(struct work_struct *work);
 123
 124 /**
 125  * Two notifier lists: the "policy" list is involved in the
 126  * validation process for a new CPU frequency policy; the
 127  * "transition" list for kernel code that needs to handle
 128  * changes to devices when the CPU clock speed changes.
 129  * The mutex locks both lists.
 130  */
 131 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 132 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 133
 134 static bool init_cpufreq_transition_notifier_list_called;
 135 static int __init init_cpufreq_transition_notifier_list(void)
 136 {
 137         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 138         init_cpufreq_transition_notifier_list_called = true;
 139         return 0;
 140 }
 141 pure_initcall(init_cpufreq_transition_notifier_list);
 142
 143 static int off __read_mostly;
 144 static int cpufreq_disabled(void)
 145 {
 146         return off;
 147 }
 148 void disable_cpufreq(void)
 149 {
 150         off = 1;
 151 }
 152 static DEFINE_MUTEX(cpufreq_governor_mutex);
 153
 154 bool have_governor_per_policy(void)
 155 {
 156         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 157 }
 158 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 159
 160 bool cpufreq_driver_is_slow(void)
 161 {
 162         return !(cpufreq_driver->flags & CPUFREQ_DRIVER_FAST);
 163 }
 164 EXPORT_SYMBOL_GPL(cpufreq_driver_is_slow);
 165
 166 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 167 {
 168         if (have_governor_per_policy())
 169                 return &policy->kobj;
 170         else
 171                 return cpufreq_global_kobject;
 172 }
 173 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 174
 175 struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu)
 176 {
 177         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 178
 179         return policy && !policy_is_inactive(policy) ?
 180                 policy->freq_table : NULL;
 181 }
 182 EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table);
 183
 184 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 185 {
 186         u64 idle_time;
 187         u64 cur_wall_time;
 188         u64 busy_time;
 189
 190         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 191
 192         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 193         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 194         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 195         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 196         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 197         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 198
 199         idle_time = cur_wall_time - busy_time;
 200         if (wall)
 201                 *wall = cputime_to_usecs(cur_wall_time);
 202
 203         return cputime_to_usecs(idle_time);
 204 }
 205
 206 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 207 {
 208         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 209
 210         if (idle_time == -1ULL)
 211                 return get_cpu_idle_time_jiffy(cpu, wall);
 212         else if (!io_busy)
 213                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 214
 215         return idle_time;
 216 }
 217 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 218
 219 /*
 220  * This is a generic cpufreq init() routine which can be used by cpufreq
 221  * drivers of SMP systems. It will do following:
 222  * - validate & show freq table passed
 223  * - set policies transition latency
 224  * - policy->cpus with all possible CPUs
 225  */
 226 int cpufreq_generic_init(struct cpufreq_policy *policy,
 227                 struct cpufreq_frequency_table *table,
 228                 unsigned int transition_latency)
 229 {
 230         int ret;
 231
 232         ret = cpufreq_table_validate_and_show(policy, table);
 233         if (ret) {
 234                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 235                 return ret;
 236         }
 237
 238         policy->cpuinfo.transition_latency = transition_latency;
 239
 240         /*
 241          * The driver only supports the SMP configuration where all processors
 242          * share the clock and voltage and clock.
 243          */
 244         cpumask_setall(policy->cpus);
 245
 246         return 0;
 247 }
 248 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 249
 250 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 251 {
 252         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 253
 254         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 255 }
 256 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
 257
 258 unsigned int cpufreq_generic_get(unsigned int cpu)
 259 {
 260         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 261
 262         if (!policy || IS_ERR(policy->clk)) {
 263                 pr_err("%s: No %s associated to cpu: %d\n",
 264                        __func__, policy ? "clk" : "policy", cpu);
 265                 return 0;
 266         }
 267
 268         return clk_get_rate(policy->clk) / 1000;
 269 }
 270 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 271
 272 /**
 273  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 274  *
 275  * @cpu: cpu to find policy for.
 276  *
 277  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 278  * It also increments the kobject reference count to mark it busy and so would
 279  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 280  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 281  * freed as that depends on the kobj count.
 282  *
 283  * Return: A valid policy on success, otherwise NULL on failure.
 284  */
 285 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 286 {
 287         struct cpufreq_policy *policy = NULL;
 288         unsigned long flags;
 289
 290         if (WARN_ON(cpu >= nr_cpu_ids))
 291                 return NULL;
 292
 293         /* get the cpufreq driver */
 294         read_lock_irqsave(&cpufreq_driver_lock, flags);
 295
 296         if (cpufreq_driver) {
 297                 /* get the CPU */
 298                 policy = cpufreq_cpu_get_raw(cpu);
 299                 if (policy)
 300                         kobject_get(&policy->kobj);
 301         }
 302
 303         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 304
 305         return policy;
 306 }
 307 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 308
 309 /**
 310  * cpufreq_cpu_put: Decrements the usage count of a policy
 311  *
 312  * @policy: policy earlier returned by cpufreq_cpu_get().
 313  *
 314  * This decrements the kobject reference count incremented earlier by calling
 315  * cpufreq_cpu_get().
 316  */
 317 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 318 {
 319         kobject_put(&policy->kobj);
 320 }
 321 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 322
 323 /*********************************************************************
 324  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 325  *********************************************************************/
 326
 327 /**
 328  * adjust_jiffies - adjust the system "loops_per_jiffy"
 329  *
 330  * This function alters the system "loops_per_jiffy" for the clock
 331  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 332  * systems as each CPU might be scaled differently. So, use the arch
 333  * per-CPU loops_per_jiffy value wherever possible.
 334  */
 335 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 336 {
 337 #ifndef CONFIG_SMP
 338         static unsigned long l_p_j_ref;
 339         static unsigned int l_p_j_ref_freq;
 340
 341         if (ci->flags & CPUFREQ_CONST_LOOPS)
 342                 return;
 343
 344         if (!l_p_j_ref_freq) {
 345                 l_p_j_ref = loops_per_jiffy;
 346                 l_p_j_ref_freq = ci->old;
 347                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 348                          l_p_j_ref, l_p_j_ref_freq);
 349         }
 350         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 351                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 352                                                                 ci->new);
 353                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 354                          loops_per_jiffy, ci->new);
 355         }
 356 #endif
 357 }
 358
 359 /*********************************************************************
 360  *               FREQUENCY INVARIANT CPU CAPACITY                    *
 361  *********************************************************************/
 362
 363 static DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
 364 static DEFINE_PER_CPU(unsigned long, max_freq_scale) = SCHED_CAPACITY_SCALE;
 365
 366 static void
 367 scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs)
 368 {
 369         unsigned long cur = freqs ? freqs->new : policy->cur;
 370         unsigned long scale = (cur << SCHED_CAPACITY_SHIFT) / policy->max;
 371         struct cpufreq_cpuinfo *cpuinfo = &policy->cpuinfo;
 372         int cpu;
 373
 374         pr_debug("cpus %*pbl cur/cur max freq %lu/%u kHz freq scale %lu\n",
 375                  cpumask_pr_args(policy->cpus), cur, policy->max, scale);
 376
 377         for_each_cpu(cpu, policy->cpus)
 378                 per_cpu(freq_scale, cpu) = scale;
 379
 380         if (freqs)
 381                 return;
 382
 383         scale = (policy->max << SCHED_CAPACITY_SHIFT) / cpuinfo->max_freq;
 384
 385         pr_debug("cpus %*pbl cur max/max freq %u/%u kHz max freq scale %lu\n",
 386                  cpumask_pr_args(policy->cpus), policy->max, cpuinfo->max_freq,
 387                  scale);
 388
 389         for_each_cpu(cpu, policy->cpus)
 390                 per_cpu(max_freq_scale, cpu) = scale;
 391 }
 392
 393 unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu)
 394 {
 395         return per_cpu(freq_scale, cpu);
 396 }
 397
 398 unsigned long cpufreq_scale_max_freq_capacity(int cpu)
 399 {
 400         return per_cpu(max_freq_scale, cpu);
 401 }
 402
 403 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 404                 struct cpufreq_freqs *freqs, unsigned int state)
 405 {
 406         BUG_ON(irqs_disabled());
 407
 408         if (cpufreq_disabled())
 409                 return;
 410
 411         freqs->flags = cpufreq_driver->flags;
 412         pr_debug("notification %u of frequency transition to %u kHz\n",
 413                  state, freqs->new);
 414
 415         switch (state) {
 416
 417         case CPUFREQ_PRECHANGE:
 418                 /* detect if the driver reported a value as "old frequency"
 419                  * which is not equal to what the cpufreq core thinks is
 420                  * "old frequency".
 421                  */
 422                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 423                         if ((policy) && (policy->cpu == freqs->cpu) &&
 424                             (policy->cur) && (policy->cur != freqs->old)) {
 425                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 426                                          freqs->old, policy->cur);
 427                                 freqs->old = policy->cur;
 428                         }
 429                 }
 430                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 431                                 CPUFREQ_PRECHANGE, freqs);
 432                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 433                 break;
 434
 435         case CPUFREQ_POSTCHANGE:
 436                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 437                 pr_debug("FREQ: %lu - CPU: %lu\n",
 438                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 439                 trace_cpu_frequency(freqs->new, freqs->cpu);
 440                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 441                                 CPUFREQ_POSTCHANGE, freqs);
 442                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 443                         policy->cur = freqs->new;
 444                 break;
 445         }
 446 }
 447
 448 /**
 449  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 450  * on frequency transition.
 451  *
 452  * This function calls the transition notifiers and the "adjust_jiffies"
 453  * function. It is called twice on all CPU frequency changes that have
 454  * external effects.
 455  */
 456 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 457                 struct cpufreq_freqs *freqs, unsigned int state)
 458 {
 459         for_each_cpu(freqs->cpu, policy->cpus)
 460                 __cpufreq_notify_transition(policy, freqs, state);
 461 }
 462
 463 /* Do post notifications when there are chances that transition has failed */
 464 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 465                 struct cpufreq_freqs *freqs, int transition_failed)
 466 {
 467         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 468         if (!transition_failed)
 469                 return;
 470
 471         swap(freqs->old, freqs->new);
 472         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 473         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 474 }
 475
 476 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 477                 struct cpufreq_freqs *freqs)
 478 {
 479 #ifdef CONFIG_SMP
 480         int cpu;
 481 #endif
 482
 483         /*
 484          * Catch double invocations of _begin() which lead to self-deadlock.
 485          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 486          * doesn't invoke _begin() on their behalf, and hence the chances of
 487          * double invocations are very low. Moreover, there are scenarios
 488          * where these checks can emit false-positive warnings in these
 489          * drivers; so we avoid that by skipping them altogether.
 490          */
 491         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 492                                 && current == policy->transition_task);
 493
 494 wait:
 495         wait_event(policy->transition_wait, !policy->transition_ongoing);
 496
 497         spin_lock(&policy->transition_lock);
 498
 499         if (unlikely(policy->transition_ongoing)) {
 500                 spin_unlock(&policy->transition_lock);
 501                 goto wait;
 502         }
 503
 504         policy->transition_ongoing = true;
 505         policy->transition_task = current;
 506
 507         spin_unlock(&policy->transition_lock);
 508
 509         scale_freq_capacity(policy, freqs);
 510 #ifdef CONFIG_SMP
 511         for_each_cpu(cpu, policy->cpus)
 512                 trace_cpu_capacity(capacity_curr_of(cpu), cpu);
 513 #endif
 514
 515         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 516 }
 517 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 518
 519 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 520                 struct cpufreq_freqs *freqs, int transition_failed)
 521 {
 522         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 523                 return;
 524
 525         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 526
 527         policy->transition_ongoing = false;
 528         policy->transition_task = NULL;
 529
 530         wake_up(&policy->transition_wait);
 531 }
 532 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 533
 534 /**
 535  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
 536  * one.
 537  * @target_freq: target frequency to resolve.
 538  *
 539  * The target to driver frequency mapping is cached in the policy.
 540  *
 541  * Return: Lowest driver-supported frequency greater than or equal to the
 542  * given target_freq, subject to policy (min/max) and driver limitations.
 543  */
 544 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
 545                                          unsigned int target_freq)
 546 {
 547         target_freq = clamp_val(target_freq, policy->min, policy->max);
 548         policy->cached_target_freq = target_freq;
 549
 550         if (cpufreq_driver->target_index) {
 551                 int idx, rv;
 552
 553                 rv = cpufreq_frequency_table_target(policy, policy->freq_table,
 554                                                     target_freq,
 555                                                     CPUFREQ_RELATION_L,
 556                                                     &idx);
 557                 if (rv)
 558                         return target_freq;
 559                 policy->cached_resolved_idx = idx;
 560                 return policy->freq_table[idx].frequency;
 561         }
 562
 563         return target_freq;
 564 }
 565 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
 566
 567 /*********************************************************************
 568  *                          SYSFS INTERFACE                          *
 569  *********************************************************************/
 570 static ssize_t show_boost(struct kobject *kobj,
 571                                  struct attribute *attr, char *buf)
 572 {
 573         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 574 }
 575
 576 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 577                                   const char *buf, size_t count)
 578 {
 579         int ret, enable;
 580
 581         ret = sscanf(buf, "%d", &enable);
 582         if (ret != 1 || enable < 0 || enable > 1)
 583                 return -EINVAL;
 584
 585         if (cpufreq_boost_trigger_state(enable)) {
 586                 pr_err("%s: Cannot %s BOOST!\n",
 587                        __func__, enable ? "enable" : "disable");
 588                 return -EINVAL;
 589         }
 590
 591         pr_debug("%s: cpufreq BOOST %s\n",
 592                  __func__, enable ? "enabled" : "disabled");
 593
 594         return count;
 595 }
 596 define_one_global_rw(boost);
 597
 598 static struct cpufreq_governor *find_governor(const char *str_governor)
 599 {
 600         struct cpufreq_governor *t;
 601
 602         for_each_governor(t)
 603                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 604                         return t;
 605
 606         return NULL;
 607 }
 608
 609 /**
 610  * cpufreq_parse_governor - parse a governor string
 611  */
 612 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 613                                 struct cpufreq_governor **governor)
 614 {
 615         int err = -EINVAL;
 616
 617         if (cpufreq_driver->setpolicy) {
 618                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 619                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 620                         err = 0;
 621                 } else if (!strncasecmp(str_governor, "powersave",
 622                                                 CPUFREQ_NAME_LEN)) {
 623                         *policy = CPUFREQ_POLICY_POWERSAVE;
 624                         err = 0;
 625                 }
 626         } else {
 627                 struct cpufreq_governor *t;
 628
 629                 mutex_lock(&cpufreq_governor_mutex);
 630
 631                 t = find_governor(str_governor);
 632
 633                 if (t == NULL) {
 634                         int ret;
 635
 636                         mutex_unlock(&cpufreq_governor_mutex);
 637                         ret = request_module("cpufreq_%s", str_governor);
 638                         mutex_lock(&cpufreq_governor_mutex);
 639
 640                         if (ret == 0)
 641                                 t = find_governor(str_governor);
 642                 }
 643
 644                 if (t != NULL) {
 645                         *governor = t;
 646                         err = 0;
 647                 }
 648
 649                 mutex_unlock(&cpufreq_governor_mutex);
 650         }
 651         return err;
 652 }
 653
 654 /**
 655  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 656  * print out cpufreq information
 657  *
 658  * Write out information from cpufreq_driver->policy[cpu]; object must be
 659  * "unsigned int".
 660  */
 661
 662 #define show_one(file_name, object)                     \
 663 static ssize_t show_##file_name                         \
 664 (struct cpufreq_policy *policy, char *buf)              \
 665 {                                                       \
 666         return sprintf(buf, "%u\n", policy->object);    \
 667 }
 668
 669 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 670 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 671 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 672 show_one(scaling_min_freq, min);
 673 show_one(scaling_max_freq, max);
 674
 675 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 676 {
 677         ssize_t ret;
 678
 679         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 680                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 681         else
 682                 ret = sprintf(buf, "%u\n", policy->cur);
 683         return ret;
 684 }
 685
 686 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 687                                 struct cpufreq_policy *new_policy);
 688
 689 /**
 690  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 691  */
 692 #define store_one(file_name, object)                    \
 693 static ssize_t store_##file_name                                        \
 694 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 695 {                                                                       \
 696         int ret, temp;                                                  \
 697         struct cpufreq_policy new_policy;                               \
 698                                                                         \
 699         memcpy(&new_policy, policy, sizeof(*policy));                   \
 700                                                                         \
 701         ret = sscanf(buf, "%u", &new_policy.object);                    \
 702         if (ret != 1)                                                   \
 703                 return -EINVAL;                                         \
 704                                                                         \
 705         temp = new_policy.object;                                       \
 706         ret = cpufreq_set_policy(policy, &new_policy);          \
 707         if (!ret)                                                       \
 708                 policy->user_policy.object = temp;                      \
 709                                                                         \
 710         return ret ? ret : count;                                       \
 711 }
 712
 713 store_one(scaling_min_freq, min);
 714 store_one(scaling_max_freq, max);
 715
 716 /**
 717  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 718  */
 719 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 720                                         char *buf)
 721 {
 722         unsigned int cur_freq = __cpufreq_get(policy);
 723
 724         if (cur_freq)
 725                 return sprintf(buf, "%u\n", cur_freq);
 726
 727         return sprintf(buf, "<unknown>\n");
 728 }
 729
 730 /**
 731  * show_scaling_governor - show the current policy for the specified CPU
 732  */
 733 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 734 {
 735         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 736                 return sprintf(buf, "powersave\n");
 737         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 738                 return sprintf(buf, "performance\n");
 739         else if (policy->governor)
 740                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 741                                 policy->governor->name);
 742         return -EINVAL;
 743 }
 744
 745 /**
 746  * store_scaling_governor - store policy for the specified CPU
 747  */
 748 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 749                                         const char *buf, size_t count)
 750 {
 751         int ret;
 752         char    str_governor[16];
 753         struct cpufreq_policy new_policy;
 754
 755         memcpy(&new_policy, policy, sizeof(*policy));
 756
 757         ret = sscanf(buf, "%15s", str_governor);
 758         if (ret != 1)
 759                 return -EINVAL;
 760
 761         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 762                                                 &new_policy.governor))
 763                 return -EINVAL;
 764
 765         ret = cpufreq_set_policy(policy, &new_policy);
 766         return ret ? ret : count;
 767 }
 768
 769 /**
 770  * show_scaling_driver - show the cpufreq driver currently loaded
 771  */
 772 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 773 {
 774         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 775 }
 776
 777 /**
 778  * show_scaling_available_governors - show the available CPUfreq governors
 779  */
 780 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 781                                                 char *buf)
 782 {
 783         ssize_t i = 0;
 784         struct cpufreq_governor *t;
 785
 786         if (!has_target()) {
 787                 i += sprintf(buf, "performance powersave");
 788                 goto out;
 789         }
 790
 791         for_each_governor(t) {
 792                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 793                     - (CPUFREQ_NAME_LEN + 2)))
 794                         goto out;
 795                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 796         }
 797 out:
 798         i += sprintf(&buf[i], "\n");
 799         return i;
 800 }
 801
 802 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 803 {
 804         ssize_t i = 0;
 805         unsigned int cpu;
 806
 807         for_each_cpu(cpu, mask) {
 808                 if (i)
 809                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 810                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 811                 if (i >= (PAGE_SIZE - 5))
 812                         break;
 813         }
 814         i += sprintf(&buf[i], "\n");
 815         return i;
 816 }
 817 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 818
 819 /**
 820  * show_related_cpus - show the CPUs affected by each transition even if
 821  * hw coordination is in use
 822  */
 823 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 824 {
 825         return cpufreq_show_cpus(policy->related_cpus, buf);
 826 }
 827
 828 /**
 829  * show_affected_cpus - show the CPUs affected by each transition
 830  */
 831 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 832 {
 833         return cpufreq_show_cpus(policy->cpus, buf);
 834 }
 835
 836 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 837                                         const char *buf, size_t count)
 838 {
 839         unsigned int freq = 0;
 840         unsigned int ret;
 841
 842         if (!policy->governor || !policy->governor->store_setspeed)
 843                 return -EINVAL;
 844
 845         ret = sscanf(buf, "%u", &freq);
 846         if (ret != 1)
 847                 return -EINVAL;
 848
 849         policy->governor->store_setspeed(policy, freq);
 850
 851         return count;
 852 }
 853
 854 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 855 {
 856         if (!policy->governor || !policy->governor->show_setspeed)
 857                 return sprintf(buf, "<unsupported>\n");
 858
 859         return policy->governor->show_setspeed(policy, buf);
 860 }
 861
 862 /**
 863  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 864  */
 865 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 866 {
 867         unsigned int limit;
 868         int ret;
 869         if (cpufreq_driver->bios_limit) {
 870                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 871                 if (!ret)
 872                         return sprintf(buf, "%u\n", limit);
 873         }
 874         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 875 }
 876
 877 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 878 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 879 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 880 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 881 cpufreq_freq_attr_ro(scaling_available_governors);
 882 cpufreq_freq_attr_ro(scaling_driver);
 883 cpufreq_freq_attr_ro(scaling_cur_freq);
 884 cpufreq_freq_attr_ro(bios_limit);
 885 cpufreq_freq_attr_ro(related_cpus);
 886 cpufreq_freq_attr_ro(affected_cpus);
 887 cpufreq_freq_attr_rw(scaling_min_freq);
 888 cpufreq_freq_attr_rw(scaling_max_freq);
 889 cpufreq_freq_attr_rw(scaling_governor);
 890 cpufreq_freq_attr_rw(scaling_setspeed);
 891
 892 static struct attribute *default_attrs[] = {
 893         &cpuinfo_min_freq.attr,
 894         &cpuinfo_max_freq.attr,
 895         &cpuinfo_transition_latency.attr,
 896         &scaling_min_freq.attr,
 897         &scaling_max_freq.attr,
 898         &affected_cpus.attr,
 899         &related_cpus.attr,
 900         &scaling_governor.attr,
 901         &scaling_driver.attr,
 902         &scaling_available_governors.attr,
 903         &scaling_setspeed.attr,
 904         NULL
 905 };
 906
 907 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 908 #define to_attr(a) container_of(a, struct freq_attr, attr)
 909
 910 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 911 {
 912         struct cpufreq_policy *policy = to_policy(kobj);
 913         struct freq_attr *fattr = to_attr(attr);
 914         ssize_t ret;
 915
 916         down_read(&policy->rwsem);
 917
 918         if (fattr->show)
 919                 ret = fattr->show(policy, buf);
 920         else
 921                 ret = -EIO;
 922
 923         up_read(&policy->rwsem);
 924
 925         return ret;
 926 }
 927
 928 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 929                      const char *buf, size_t count)
 930 {
 931         struct cpufreq_policy *policy = to_policy(kobj);
 932         struct freq_attr *fattr = to_attr(attr);
 933         ssize_t ret = -EINVAL;
 934
 935         get_online_cpus();
 936
 937         if (!cpu_online(policy->cpu))
 938                 goto unlock;
 939
 940         down_write(&policy->rwsem);
 941
 942         if (fattr->store)
 943                 ret = fattr->store(policy, buf, count);
 944         else
 945                 ret = -EIO;
 946
 947         up_write(&policy->rwsem);
 948 unlock:
 949         put_online_cpus();
 950
 951         return ret;
 952 }
 953
 954 static void cpufreq_sysfs_release(struct kobject *kobj)
 955 {
 956         struct cpufreq_policy *policy = to_policy(kobj);
 957         pr_debug("last reference is dropped\n");
 958         complete(&policy->kobj_unregister);
 959 }
 960
 961 static const struct sysfs_ops sysfs_ops = {
 962         .show   = show,
 963         .store  = store,
 964 };
 965
 966 static struct kobj_type ktype_cpufreq = {
 967         .sysfs_ops      = &sysfs_ops,
 968         .default_attrs  = default_attrs,
 969         .release        = cpufreq_sysfs_release,
 970 };
 971
 972 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 973 {
 974         struct device *cpu_dev;
 975
 976         pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
 977
 978         if (!policy)
 979                 return 0;
 980
 981         cpu_dev = get_cpu_device(cpu);
 982         if (WARN_ON(!cpu_dev))
 983                 return 0;
 984
 985         return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
 986 }
 987
 988 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 989 {
 990         struct device *cpu_dev;
 991
 992         pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
 993
 994         cpu_dev = get_cpu_device(cpu);
 995         if (WARN_ON(!cpu_dev))
 996                 return;
 997
 998         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
 999 }
1000
1001 /* Add/remove symlinks for all related CPUs */
1002 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
1003 {
1004         unsigned int j;
1005         int ret = 0;
1006
1007         /* Some related CPUs might not be present (physically hotplugged) */
1008         for_each_cpu(j, policy->real_cpus) {
1009                 ret = add_cpu_dev_symlink(policy, j);
1010                 if (ret)
1011                         break;
1012         }
1013
1014         return ret;
1015 }
1016
1017 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
1018 {
1019         unsigned int j;
1020
1021         /* Some related CPUs might not be present (physically hotplugged) */
1022         for_each_cpu(j, policy->real_cpus)
1023                 remove_cpu_dev_symlink(policy, j);
1024 }
1025
1026 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1027 {
1028         struct freq_attr **drv_attr;
1029         int ret = 0;
1030
1031         /* set up files for this cpu device */
1032         drv_attr = cpufreq_driver->attr;
1033         while (drv_attr && *drv_attr) {
1034                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1035                 if (ret)
1036                         return ret;
1037                 drv_attr++;
1038         }
1039         if (cpufreq_driver->get) {
1040                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1041                 if (ret)
1042                         return ret;
1043         }
1044
1045         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1046         if (ret)
1047                 return ret;
1048
1049         if (cpufreq_driver->bios_limit) {
1050                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1051                 if (ret)
1052                         return ret;
1053         }
1054
1055         return cpufreq_add_dev_symlink(policy);
1056 }
1057
1058 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1059 {
1060         struct cpufreq_governor *gov = NULL;
1061         struct cpufreq_policy new_policy;
1062
1063         memcpy(&new_policy, policy, sizeof(*policy));
1064
1065         /* Update governor of new_policy to the governor used before hotplug */
1066         gov = find_governor(policy->last_governor);
1067         if (gov)
1068                 pr_debug("Restoring governor %s for cpu %d\n",
1069                                 policy->governor->name, policy->cpu);
1070         else
1071                 gov = CPUFREQ_DEFAULT_GOVERNOR;
1072
1073         new_policy.governor = gov;
1074
1075         /* Use the default policy if there is no last_policy. */
1076         if (cpufreq_driver->setpolicy) {
1077                 if (policy->last_policy)
1078                         new_policy.policy = policy->last_policy;
1079                 else
1080                         cpufreq_parse_governor(gov->name, &new_policy.policy,
1081                                                NULL);
1082         }
1083         /* set default policy */
1084         return cpufreq_set_policy(policy, &new_policy);
1085 }
1086
1087 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1088 {
1089         int ret = 0;
1090
1091         /* Has this CPU been taken care of already? */
1092         if (cpumask_test_cpu(cpu, policy->cpus))
1093                 return 0;
1094
1095         if (has_target()) {
1096                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1097                 if (ret) {
1098                         pr_err("%s: Failed to stop governor\n", __func__);
1099                         return ret;
1100                 }
1101         }
1102
1103         down_write(&policy->rwsem);
1104         cpumask_set_cpu(cpu, policy->cpus);
1105         up_write(&policy->rwsem);
1106
1107         if (has_target()) {
1108                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1109                 if (!ret)
1110                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1111
1112                 if (ret) {
1113                         pr_err("%s: Failed to start governor\n", __func__);
1114                         return ret;
1115                 }
1116         }
1117
1118         return 0;
1119 }
1120
1121 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1122 {
1123         struct device *dev = get_cpu_device(cpu);
1124         struct cpufreq_policy *policy;
1125
1126         if (WARN_ON(!dev))
1127                 return NULL;
1128
1129         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1130         if (!policy)
1131                 return NULL;
1132
1133         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1134                 goto err_free_policy;
1135
1136         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1137                 goto err_free_cpumask;
1138
1139         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1140                 goto err_free_rcpumask;
1141
1142         kobject_init(&policy->kobj, &ktype_cpufreq);
1143         INIT_LIST_HEAD(&policy->policy_list);
1144         init_rwsem(&policy->rwsem);
1145         spin_lock_init(&policy->transition_lock);
1146         init_waitqueue_head(&policy->transition_wait);
1147         init_completion(&policy->kobj_unregister);
1148         INIT_WORK(&policy->update, handle_update);
1149
1150         policy->cpu = cpu;
1151         return policy;
1152
1153 err_free_rcpumask:
1154         free_cpumask_var(policy->related_cpus);
1155 err_free_cpumask:
1156         free_cpumask_var(policy->cpus);
1157 err_free_policy:
1158         kfree(policy);
1159
1160         return NULL;
1161 }
1162
1163 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1164 {
1165         struct kobject *kobj;
1166         struct completion *cmp;
1167
1168         if (notify)
1169                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1170                                              CPUFREQ_REMOVE_POLICY, policy);
1171
1172         down_write(&policy->rwsem);
1173         cpufreq_remove_dev_symlink(policy);
1174         kobj = &policy->kobj;
1175         cmp = &policy->kobj_unregister;
1176         up_write(&policy->rwsem);
1177         kobject_put(kobj);
1178
1179         /*
1180          * We need to make sure that the underlying kobj is
1181          * actually not referenced anymore by anybody before we
1182          * proceed with unloading.
1183          */
1184         pr_debug("waiting for dropping of refcount\n");
1185         wait_for_completion(cmp);
1186         pr_debug("wait complete\n");
1187 }
1188
1189 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1190 {
1191         unsigned long flags;
1192         int cpu;
1193
1194         /* Remove policy from list */
1195         write_lock_irqsave(&cpufreq_driver_lock, flags);
1196         list_del(&policy->policy_list);
1197
1198         for_each_cpu(cpu, policy->related_cpus)
1199                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1200         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1201
1202         cpufreq_policy_put_kobj(policy, notify);
1203         free_cpumask_var(policy->real_cpus);
1204         free_cpumask_var(policy->related_cpus);
1205         free_cpumask_var(policy->cpus);
1206         kfree(policy);
1207 }
1208
1209 static int cpufreq_online(unsigned int cpu)
1210 {
1211         struct cpufreq_policy *policy;
1212         bool new_policy;
1213         unsigned long flags;
1214         unsigned int j;
1215         int ret;
1216
1217         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1218
1219         /* Check if this CPU already has a policy to manage it */
1220         policy = per_cpu(cpufreq_cpu_data, cpu);
1221         if (policy) {
1222                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1223                 if (!policy_is_inactive(policy))
1224                         return cpufreq_add_policy_cpu(policy, cpu);
1225
1226                 /* This is the only online CPU for the policy.  Start over. */
1227                 new_policy = false;
1228                 down_write(&policy->rwsem);
1229                 policy->cpu = cpu;
1230                 policy->governor = NULL;
1231                 up_write(&policy->rwsem);
1232         } else {
1233                 new_policy = true;
1234                 policy = cpufreq_policy_alloc(cpu);
1235                 if (!policy)
1236                         return -ENOMEM;
1237         }
1238
1239         cpumask_copy(policy->cpus, cpumask_of(cpu));
1240
1241         /* call driver. From then on the cpufreq must be able
1242          * to accept all calls to ->verify and ->setpolicy for this CPU
1243          */
1244         ret = cpufreq_driver->init(policy);
1245         if (ret) {
1246                 pr_debug("initialization failed\n");
1247                 goto out_free_policy;
1248         }
1249
1250         down_write(&policy->rwsem);
1251
1252         if (new_policy) {
1253                 /* related_cpus should at least include policy->cpus. */
1254                 cpumask_copy(policy->related_cpus, policy->cpus);
1255                 /* Remember CPUs present at the policy creation time. */
1256                 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
1257
1258                 /* Name and add the kobject */
1259                 ret = kobject_add(&policy->kobj, cpufreq_global_kobject,
1260                                   "policy%u",
1261                                   cpumask_first(policy->related_cpus));
1262                 if (ret) {
1263                         pr_err("%s: failed to add policy->kobj: %d\n", __func__,
1264                                ret);
1265                         goto out_exit_policy;
1266                 }
1267         }
1268
1269         /*
1270          * affected cpus must always be the one, which are online. We aren't
1271          * managing offline cpus here.
1272          */
1273         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1274
1275         if (new_policy) {
1276                 policy->user_policy.min = policy->min;
1277                 policy->user_policy.max = policy->max;
1278
1279                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1280                 for_each_cpu(j, policy->related_cpus)
1281                         per_cpu(cpufreq_cpu_data, j) = policy;
1282                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1283         } else {
1284                 policy->min = policy->user_policy.min;
1285                 policy->max = policy->user_policy.max;
1286         }
1287
1288         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1289                 policy->cur = cpufreq_driver->get(policy->cpu);
1290                 if (!policy->cur) {
1291                         pr_err("%s: ->get() failed\n", __func__);
1292                         goto out_exit_policy;
1293                 }
1294         }
1295
1296         /*
1297          * Sometimes boot loaders set CPU frequency to a value outside of
1298          * frequency table present with cpufreq core. In such cases CPU might be
1299          * unstable if it has to run on that frequency for long duration of time
1300          * and so its better to set it to a frequency which is specified in
1301          * freq-table. This also makes cpufreq stats inconsistent as
1302          * cpufreq-stats would fail to register because current frequency of CPU
1303          * isn't found in freq-table.
1304          *
1305          * Because we don't want this change to effect boot process badly, we go
1306          * for the next freq which is >= policy->cur ('cur' must be set by now,
1307          * otherwise we will end up setting freq to lowest of the table as 'cur'
1308          * is initialized to zero).
1309          *
1310          * We are passing target-freq as "policy->cur - 1" otherwise
1311          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1312          * equal to target-freq.
1313          */
1314         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1315             && has_target()) {
1316                 /* Are we running at unknown frequency ? */
1317                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1318                 if (ret == -EINVAL) {
1319                         /* Warn user and fix it */
1320                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1321                                 __func__, policy->cpu, policy->cur);
1322                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1323                                 CPUFREQ_RELATION_L);
1324
1325                         /*
1326                          * Reaching here after boot in a few seconds may not
1327                          * mean that system will remain stable at "unknown"
1328                          * frequency for longer duration. Hence, a BUG_ON().
1329                          */
1330                         BUG_ON(ret);
1331                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1332                                 __func__, policy->cpu, policy->cur);
1333                 }
1334         }
1335
1336         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1337                                      CPUFREQ_START, policy);
1338
1339         if (new_policy) {
1340                 ret = cpufreq_add_dev_interface(policy);
1341                 if (ret)
1342                         goto out_exit_policy;
1343                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1344                                 CPUFREQ_CREATE_POLICY, policy);
1345
1346                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1347                 list_add(&policy->policy_list, &cpufreq_policy_list);
1348                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1349         }
1350
1351         ret = cpufreq_init_policy(policy);
1352         if (ret) {
1353                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1354                        __func__, cpu, ret);
1355                 /* cpufreq_policy_free() will notify based on this */
1356                 new_policy = false;
1357                 goto out_exit_policy;
1358         }
1359
1360         up_write(&policy->rwsem);
1361
1362         kobject_uevent(&policy->kobj, KOBJ_ADD);
1363
1364         /* Callback for handling stuff after policy is ready */
1365         if (cpufreq_driver->ready)
1366                 cpufreq_driver->ready(policy);
1367
1368         pr_debug("initialization complete\n");
1369
1370         return 0;
1371
1372 out_exit_policy:
1373         up_write(&policy->rwsem);
1374
1375         if (cpufreq_driver->exit)
1376                 cpufreq_driver->exit(policy);
1377 out_free_policy:
1378         cpufreq_policy_free(policy, !new_policy);
1379         return ret;
1380 }
1381
1382 /**
1383  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1384  * @dev: CPU device.
1385  * @sif: Subsystem interface structure pointer (not used)
1386  */
1387 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1388 {
1389         unsigned cpu = dev->id;
1390         int ret;
1391
1392         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1393
1394         if (cpu_online(cpu)) {
1395                 ret = cpufreq_online(cpu);
1396         } else {
1397                 /*
1398                  * A hotplug notifier will follow and we will handle it as CPU
1399                  * online then.  For now, just create the sysfs link, unless
1400                  * there is no policy or the link is already present.
1401                  */
1402                 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1403
1404                 ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)
1405                         ? add_cpu_dev_symlink(policy, cpu) : 0;
1406         }
1407
1408         return ret;
1409 }
1410
1411 static void cpufreq_offline_prepare(unsigned int cpu)
1412 {
1413         struct cpufreq_policy *policy;
1414
1415         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1416
1417         policy = cpufreq_cpu_get_raw(cpu);
1418         if (!policy) {
1419                 pr_debug("%s: No cpu_data found\n", __func__);
1420                 return;
1421         }
1422
1423         if (has_target()) {
1424                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1425                 if (ret)
1426                         pr_err("%s: Failed to stop governor\n", __func__);
1427         }
1428
1429         down_write(&policy->rwsem);
1430         cpumask_clear_cpu(cpu, policy->cpus);
1431
1432         if (policy_is_inactive(policy)) {
1433                 if (has_target())
1434                         strncpy(policy->last_governor, policy->governor->name,
1435                                 CPUFREQ_NAME_LEN);
1436                 else
1437                         policy->last_policy = policy->policy;
1438         } else if (cpu == policy->cpu) {
1439                 /* Nominate new CPU */
1440                 policy->cpu = cpumask_any(policy->cpus);
1441         }
1442         up_write(&policy->rwsem);
1443
1444         /* Start governor again for active policy */
1445         if (!policy_is_inactive(policy)) {
1446                 if (has_target()) {
1447                         int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1448                         if (!ret)
1449                                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1450
1451                         if (ret)
1452                                 pr_err("%s: Failed to start governor\n", __func__);
1453                 }
1454         } else if (cpufreq_driver->stop_cpu) {
1455                 cpufreq_driver->stop_cpu(policy);
1456         }
1457 }
1458
1459 static void cpufreq_offline_finish(unsigned int cpu)
1460 {
1461         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1462
1463         if (!policy) {
1464                 pr_debug("%s: No cpu_data found\n", __func__);
1465                 return;
1466         }
1467
1468         /* Only proceed for inactive policies */
1469         if (!policy_is_inactive(policy))
1470                 return;
1471
1472         /* If cpu is last user of policy, free policy */
1473         if (has_target()) {
1474                 int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1475                 if (ret)
1476                         pr_err("%s: Failed to exit governor\n", __func__);
1477         }
1478
1479         /*
1480          * Perform the ->exit() even during light-weight tear-down,
1481          * since this is a core component, and is essential for the
1482          * subsequent light-weight ->init() to succeed.
1483          */
1484         if (cpufreq_driver->exit) {
1485                 cpufreq_driver->exit(policy);
1486                 policy->freq_table = NULL;
1487         }
1488 }
1489
1490 /**
1491  * cpufreq_remove_dev - remove a CPU device
1492  *
1493  * Removes the cpufreq interface for a CPU device.
1494  */
1495 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1496 {
1497         unsigned int cpu = dev->id;
1498         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1499
1500         if (!policy)
1501                 return;
1502
1503         if (cpu_online(cpu)) {
1504                 cpufreq_offline_prepare(cpu);
1505                 cpufreq_offline_finish(cpu);
1506         }
1507
1508         cpumask_clear_cpu(cpu, policy->real_cpus);
1509         remove_cpu_dev_symlink(policy, cpu);
1510
1511         if (cpumask_empty(policy->real_cpus))
1512                 cpufreq_policy_free(policy, true);
1513 }
1514
1515 static void handle_update(struct work_struct *work)
1516 {
1517         struct cpufreq_policy *policy =
1518                 container_of(work, struct cpufreq_policy, update);
1519         unsigned int cpu = policy->cpu;
1520         pr_debug("handle_update for cpu %u called\n", cpu);
1521         cpufreq_update_policy(cpu);
1522 }
1523
1524 /**
1525  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1526  *      in deep trouble.
1527  *      @policy: policy managing CPUs
1528  *      @new_freq: CPU frequency the CPU actually runs at
1529  *
1530  *      We adjust to current frequency first, and need to clean up later.
1531  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1532  */
1533 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1534                                 unsigned int new_freq)
1535 {
1536         struct cpufreq_freqs freqs;
1537
1538         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1539                  policy->cur, new_freq);
1540
1541         freqs.old = policy->cur;
1542         freqs.new = new_freq;
1543
1544         cpufreq_freq_transition_begin(policy, &freqs);
1545         cpufreq_freq_transition_end(policy, &freqs, 0);
1546 }
1547
1548 /**
1549  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1550  * @cpu: CPU number
1551  *
1552  * This is the last known freq, without actually getting it from the driver.
1553  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1554  */
1555 unsigned int cpufreq_quick_get(unsigned int cpu)
1556 {
1557         struct cpufreq_policy *policy;
1558         unsigned int ret_freq = 0;
1559
1560         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1561                 return cpufreq_driver->get(cpu);
1562
1563         policy = cpufreq_cpu_get(cpu);
1564         if (policy) {
1565                 ret_freq = policy->cur;
1566                 cpufreq_cpu_put(policy);
1567         }
1568
1569         return ret_freq;
1570 }
1571 EXPORT_SYMBOL(cpufreq_quick_get);
1572
1573 /**
1574  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1575  * @cpu: CPU number
1576  *
1577  * Just return the max possible frequency for a given CPU.
1578  */
1579 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1580 {
1581         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1582         unsigned int ret_freq = 0;
1583
1584         if (policy) {
1585                 ret_freq = policy->max;
1586                 cpufreq_cpu_put(policy);
1587         }
1588
1589         return ret_freq;
1590 }
1591 EXPORT_SYMBOL(cpufreq_quick_get_max);
1592
1593 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1594 {
1595         unsigned int ret_freq = 0;
1596
1597         if (!cpufreq_driver->get)
1598                 return ret_freq;
1599
1600         ret_freq = cpufreq_driver->get(policy->cpu);
1601
1602         /* Updating inactive policies is invalid, so avoid doing that. */
1603         if (unlikely(policy_is_inactive(policy)))
1604                 return ret_freq;
1605
1606         if (ret_freq && policy->cur &&
1607                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1608                 /* verify no discrepancy between actual and
1609                                         saved value exists */
1610                 if (unlikely(ret_freq != policy->cur)) {
1611                         cpufreq_out_of_sync(policy, ret_freq);
1612                         schedule_work(&policy->update);
1613                 }
1614         }
1615
1616         return ret_freq;
1617 }
1618
1619 /**
1620  * cpufreq_get - get the current CPU frequency (in kHz)
1621  * @cpu: CPU number
1622  *
1623  * Get the CPU current (static) CPU frequency
1624  */
1625 unsigned int cpufreq_get(unsigned int cpu)
1626 {
1627         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1628         unsigned int ret_freq = 0;
1629
1630         if (policy) {
1631                 down_read(&policy->rwsem);
1632                 ret_freq = __cpufreq_get(policy);
1633                 up_read(&policy->rwsem);
1634
1635                 cpufreq_cpu_put(policy);
1636         }
1637
1638         return ret_freq;
1639 }
1640 EXPORT_SYMBOL(cpufreq_get);
1641
1642 static struct subsys_interface cpufreq_interface = {
1643         .name           = "cpufreq",
1644         .subsys         = &cpu_subsys,
1645         .add_dev        = cpufreq_add_dev,
1646         .remove_dev     = cpufreq_remove_dev,
1647 };
1648
1649 /*
1650  * In case platform wants some specific frequency to be configured
1651  * during suspend..
1652  */
1653 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1654 {
1655         int ret;
1656
1657         if (!policy->suspend_freq) {
1658                 pr_debug("%s: suspend_freq not defined\n", __func__);
1659                 return 0;
1660         }
1661
1662         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1663                         policy->suspend_freq);
1664
1665         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1666                         CPUFREQ_RELATION_H);
1667         if (ret)
1668                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1669                                 __func__, policy->suspend_freq, ret);
1670
1671         return ret;
1672 }
1673 EXPORT_SYMBOL(cpufreq_generic_suspend);
1674
1675 /**
1676  * cpufreq_suspend() - Suspend CPUFreq governors
1677  *
1678  * Called during system wide Suspend/Hibernate cycles for suspending governors
1679  * as some platforms can't change frequency after this point in suspend cycle.
1680  * Because some of the devices (like: i2c, regulators, etc) they use for
1681  * changing frequency are suspended quickly after this point.
1682  */
1683 void cpufreq_suspend(void)
1684 {
1685         struct cpufreq_policy *policy;
1686
1687         if (!cpufreq_driver)
1688                 return;
1689
1690         if (!has_target())
1691                 goto suspend;
1692
1693         pr_debug("%s: Suspending Governors\n", __func__);
1694
1695         for_each_active_policy(policy) {
1696                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1697                         pr_err("%s: Failed to stop governor for policy: %p\n",
1698                                 __func__, policy);
1699                 else if (cpufreq_driver->suspend
1700                     && cpufreq_driver->suspend(policy))
1701                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1702                                 policy);
1703         }
1704
1705 suspend:
1706         cpufreq_suspended = true;
1707 }
1708
1709 /**
1710  * cpufreq_resume() - Resume CPUFreq governors
1711  *
1712  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1713  * are suspended with cpufreq_suspend().
1714  */
1715 void cpufreq_resume(void)
1716 {
1717         struct cpufreq_policy *policy;
1718
1719         if (!cpufreq_driver)
1720                 return;
1721
1722         cpufreq_suspended = false;
1723
1724         if (!has_target())
1725                 return;
1726
1727         pr_debug("%s: Resuming Governors\n", __func__);
1728
1729         for_each_active_policy(policy) {
1730                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1731                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1732                                 policy);
1733                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1734                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1735                         pr_err("%s: Failed to start governor for policy: %p\n",
1736                                 __func__, policy);
1737         }
1738
1739         /*
1740          * schedule call cpufreq_update_policy() for first-online CPU, as that
1741          * wouldn't be hotplugged-out on suspend. It will verify that the
1742          * current freq is in sync with what we believe it to be.
1743          */
1744         policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
1745         if (WARN_ON(!policy))
1746                 return;
1747
1748         schedule_work(&policy->update);
1749 }
1750
1751 /**
1752  *      cpufreq_get_current_driver - return current driver's name
1753  *
1754  *      Return the name string of the currently loaded cpufreq driver
1755  *      or NULL, if none.
1756  */
1757 const char *cpufreq_get_current_driver(void)
1758 {
1759         if (cpufreq_driver)
1760                 return cpufreq_driver->name;
1761
1762         return NULL;
1763 }
1764 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1765
1766 /**
1767  *      cpufreq_get_driver_data - return current driver data
1768  *
1769  *      Return the private data of the currently loaded cpufreq
1770  *      driver, or NULL if no cpufreq driver is loaded.
1771  */
1772 void *cpufreq_get_driver_data(void)
1773 {
1774         if (cpufreq_driver)
1775                 return cpufreq_driver->driver_data;
1776
1777         return NULL;
1778 }
1779 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1780
1781 /*********************************************************************
1782  *                     NOTIFIER LISTS INTERFACE                      *
1783  *********************************************************************/
1784
1785 /**
1786  *      cpufreq_register_notifier - register a driver with cpufreq
1787  *      @nb: notifier function to register
1788  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1789  *
1790  *      Add a driver to one of two lists: either a list of drivers that
1791  *      are notified about clock rate changes (once before and once after
1792  *      the transition), or a list of drivers that are notified about
1793  *      changes in cpufreq policy.
1794  *
1795  *      This function may sleep, and has the same return conditions as
1796  *      blocking_notifier_chain_register.
1797  */
1798 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1799 {
1800         int ret;
1801
1802         if (cpufreq_disabled())
1803                 return -EINVAL;
1804
1805         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1806
1807         switch (list) {
1808         case CPUFREQ_TRANSITION_NOTIFIER:
1809                 ret = srcu_notifier_chain_register(
1810                                 &cpufreq_transition_notifier_list, nb);
1811                 break;
1812         case CPUFREQ_POLICY_NOTIFIER:
1813                 ret = blocking_notifier_chain_register(
1814                                 &cpufreq_policy_notifier_list, nb);
1815                 break;
1816         default:
1817                 ret = -EINVAL;
1818         }
1819
1820         return ret;
1821 }
1822 EXPORT_SYMBOL(cpufreq_register_notifier);
1823
1824 /**
1825  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1826  *      @nb: notifier block to be unregistered
1827  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1828  *
1829  *      Remove a driver from the CPU frequency notifier list.
1830  *
1831  *      This function may sleep, and has the same return conditions as
1832  *      blocking_notifier_chain_unregister.
1833  */
1834 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1835 {
1836         int ret;
1837
1838         if (cpufreq_disabled())
1839                 return -EINVAL;
1840
1841         switch (list) {
1842         case CPUFREQ_TRANSITION_NOTIFIER:
1843                 ret = srcu_notifier_chain_unregister(
1844                                 &cpufreq_transition_notifier_list, nb);
1845                 break;
1846         case CPUFREQ_POLICY_NOTIFIER:
1847                 ret = blocking_notifier_chain_unregister(
1848                                 &cpufreq_policy_notifier_list, nb);
1849                 break;
1850         default:
1851                 ret = -EINVAL;
1852         }
1853
1854         return ret;
1855 }
1856 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1857
1858
1859 /*********************************************************************
1860  *                              GOVERNORS                            *
1861  *********************************************************************/
1862
1863 /* Must set freqs->new to intermediate frequency */
1864 static int __target_intermediate(struct cpufreq_policy *policy,
1865                                  struct cpufreq_freqs *freqs, int index)
1866 {
1867         int ret;
1868
1869         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1870
1871         /* We don't need to switch to intermediate freq */
1872         if (!freqs->new)
1873                 return 0;
1874
1875         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1876                  __func__, policy->cpu, freqs->old, freqs->new);
1877
1878         cpufreq_freq_transition_begin(policy, freqs);
1879         ret = cpufreq_driver->target_intermediate(policy, index);
1880         cpufreq_freq_transition_end(policy, freqs, ret);
1881
1882         if (ret)
1883                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1884                        __func__, ret);
1885
1886         return ret;
1887 }
1888
1889 static int __target_index(struct cpufreq_policy *policy,
1890                           struct cpufreq_frequency_table *freq_table, int index)
1891 {
1892         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1893         unsigned int intermediate_freq = 0;
1894         int retval = -EINVAL;
1895         bool notify;
1896
1897         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1898         if (notify) {
1899                 /* Handle switching to intermediate frequency */
1900                 if (cpufreq_driver->get_intermediate) {
1901                         retval = __target_intermediate(policy, &freqs, index);
1902                         if (retval)
1903                                 return retval;
1904
1905                         intermediate_freq = freqs.new;
1906                         /* Set old freq to intermediate */
1907                         if (intermediate_freq)
1908                                 freqs.old = freqs.new;
1909                 }
1910
1911                 freqs.new = freq_table[index].frequency;
1912                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1913                          __func__, policy->cpu, freqs.old, freqs.new);
1914
1915                 cpufreq_freq_transition_begin(policy, &freqs);
1916         }
1917
1918         retval = cpufreq_driver->target_index(policy, index);
1919         if (retval)
1920                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1921                        retval);
1922
1923         if (notify) {
1924                 cpufreq_freq_transition_end(policy, &freqs, retval);
1925
1926                 /*
1927                  * Failed after setting to intermediate freq? Driver should have
1928                  * reverted back to initial frequency and so should we. Check
1929                  * here for intermediate_freq instead of get_intermediate, in
1930                  * case we haven't switched to intermediate freq at all.
1931                  */
1932                 if (unlikely(retval && intermediate_freq)) {
1933                         freqs.old = intermediate_freq;
1934                         freqs.new = policy->restore_freq;
1935                         cpufreq_freq_transition_begin(policy, &freqs);
1936                         cpufreq_freq_transition_end(policy, &freqs, 0);
1937                 }
1938         }
1939
1940         return retval;
1941 }
1942
1943 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1944                             unsigned int target_freq,
1945                             unsigned int relation)
1946 {
1947         unsigned int old_target_freq = target_freq;
1948         int retval = -EINVAL;
1949
1950         if (cpufreq_disabled())
1951                 return -ENODEV;
1952
1953         /* Make sure that target_freq is within supported range */
1954         if (target_freq > policy->max)
1955                 target_freq = policy->max;
1956         if (target_freq < policy->min)
1957                 target_freq = policy->min;
1958
1959         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1960                  policy->cpu, target_freq, relation, old_target_freq);
1961
1962         /*
1963          * This might look like a redundant call as we are checking it again
1964          * after finding index. But it is left intentionally for cases where
1965          * exactly same freq is called again and so we can save on few function
1966          * calls.
1967          */
1968         if (target_freq == policy->cur)
1969                 return 0;
1970
1971         /* Save last value to restore later on errors */
1972         policy->restore_freq = policy->cur;
1973
1974         if (cpufreq_driver->target)
1975                 retval = cpufreq_driver->target(policy, target_freq, relation);
1976         else if (cpufreq_driver->target_index) {
1977                 struct cpufreq_frequency_table *freq_table;
1978                 int index;
1979
1980                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1981                 if (unlikely(!freq_table)) {
1982                         pr_err("%s: Unable to find freq_table\n", __func__);
1983                         goto out;
1984                 }
1985
1986                 retval = cpufreq_frequency_table_target(policy, freq_table,
1987                                 target_freq, relation, &index);
1988                 if (unlikely(retval)) {
1989                         pr_err("%s: Unable to find matching freq\n", __func__);
1990                         goto out;
1991                 }
1992
1993                 if (freq_table[index].frequency == policy->cur) {
1994                         retval = 0;
1995                         goto out;
1996                 }
1997
1998                 retval = __target_index(policy, freq_table, index);
1999         }
2000
2001 out:
2002         return retval;
2003 }
2004 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2005
2006 int cpufreq_driver_target(struct cpufreq_policy *policy,
2007                           unsigned int target_freq,
2008                           unsigned int relation)
2009 {
2010         int ret = -EINVAL;
2011
2012         down_write(&policy->rwsem);
2013
2014         ret = __cpufreq_driver_target(policy, target_freq, relation);
2015
2016         up_write(&policy->rwsem);
2017
2018         return ret;
2019 }
2020 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2021
2022 static int __cpufreq_governor(struct cpufreq_policy *policy,
2023                                         unsigned int event)
2024 {
2025         int ret;
2026
2027         /* Only must be defined when default governor is known to have latency
2028            restrictions, like e.g. conservative or ondemand.
2029            That this is the case is already ensured in Kconfig
2030         */
2031 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2032         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2033 #else
2034         struct cpufreq_governor *gov = NULL;
2035 #endif
2036
2037         /* Don't start any governor operations if we are entering suspend */
2038         if (cpufreq_suspended)
2039                 return 0;
2040         /*
2041          * Governor might not be initiated here if ACPI _PPC changed
2042          * notification happened, so check it.
2043          */
2044         if (!policy->governor)
2045                 return -EINVAL;
2046
2047         if (policy->governor->max_transition_latency &&
2048             policy->cpuinfo.transition_latency >
2049             policy->governor->max_transition_latency) {
2050                 if (!gov)
2051                         return -EINVAL;
2052                 else {
2053                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2054                                 policy->governor->name, gov->name);
2055                         policy->governor = gov;
2056                 }
2057         }
2058
2059         if (event == CPUFREQ_GOV_POLICY_INIT)
2060                 if (!try_module_get(policy->governor->owner))
2061                         return -EINVAL;
2062
2063         pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);
2064
2065         mutex_lock(&cpufreq_governor_lock);
2066         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2067             || (!policy->governor_enabled
2068             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2069                 mutex_unlock(&cpufreq_governor_lock);
2070                 return -EBUSY;
2071         }
2072
2073         if (event == CPUFREQ_GOV_STOP)
2074                 policy->governor_enabled = false;
2075         else if (event == CPUFREQ_GOV_START)
2076                 policy->governor_enabled = true;
2077
2078         mutex_unlock(&cpufreq_governor_lock);
2079
2080         ret = policy->governor->governor(policy, event);
2081
2082         if (!ret) {
2083                 if (event == CPUFREQ_GOV_POLICY_INIT)
2084                         policy->governor->initialized++;
2085                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2086                         policy->governor->initialized--;
2087         } else {
2088                 /* Restore original values */
2089                 mutex_lock(&cpufreq_governor_lock);
2090                 if (event == CPUFREQ_GOV_STOP)
2091                         policy->governor_enabled = true;
2092                 else if (event == CPUFREQ_GOV_START)
2093                         policy->governor_enabled = false;
2094                 mutex_unlock(&cpufreq_governor_lock);
2095         }
2096
2097         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2098                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2099                 module_put(policy->governor->owner);
2100
2101         return ret;
2102 }
2103
2104 int cpufreq_register_governor(struct cpufreq_governor *governor)
2105 {
2106         int err;
2107
2108         if (!governor)
2109                 return -EINVAL;
2110
2111         if (cpufreq_disabled())
2112                 return -ENODEV;
2113
2114         mutex_lock(&cpufreq_governor_mutex);
2115
2116         governor->initialized = 0;
2117         err = -EBUSY;
2118         if (!find_governor(governor->name)) {
2119                 err = 0;
2120                 list_add(&governor->governor_list, &cpufreq_governor_list);
2121         }
2122
2123         mutex_unlock(&cpufreq_governor_mutex);
2124         return err;
2125 }
2126 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2127
2128 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2129 {
2130         struct cpufreq_policy *policy;
2131         unsigned long flags;
2132
2133         if (!governor)
2134                 return;
2135
2136         if (cpufreq_disabled())
2137                 return;
2138
2139         /* clear last_governor for all inactive policies */
2140         read_lock_irqsave(&cpufreq_driver_lock, flags);
2141         for_each_inactive_policy(policy) {
2142                 if (!strcmp(policy->last_governor, governor->name)) {
2143                         policy->governor = NULL;
2144                         strcpy(policy->last_governor, "\0");
2145                 }
2146         }
2147         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2148
2149         mutex_lock(&cpufreq_governor_mutex);
2150         list_del(&governor->governor_list);
2151         mutex_unlock(&cpufreq_governor_mutex);
2152         return;
2153 }
2154 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2155
2156
2157 /*********************************************************************
2158  *                          POLICY INTERFACE                         *
2159  *********************************************************************/
2160
2161 /**
2162  * cpufreq_get_policy - get the current cpufreq_policy
2163  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2164  *      is written
2165  *
2166  * Reads the current cpufreq policy.
2167  */
2168 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2169 {
2170         struct cpufreq_policy *cpu_policy;
2171         if (!policy)
2172                 return -EINVAL;
2173
2174         cpu_policy = cpufreq_cpu_get(cpu);
2175         if (!cpu_policy)
2176                 return -EINVAL;
2177
2178         memcpy(policy, cpu_policy, sizeof(*policy));
2179
2180         cpufreq_cpu_put(cpu_policy);
2181         return 0;
2182 }
2183 EXPORT_SYMBOL(cpufreq_get_policy);
2184
2185 /*
2186  * policy : current policy.
2187  * new_policy: policy to be set.
2188  */
2189 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2190                                 struct cpufreq_policy *new_policy)
2191 {
2192         struct cpufreq_governor *old_gov;
2193         int ret;
2194
2195         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2196                  new_policy->cpu, new_policy->min, new_policy->max);
2197
2198         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2199
2200         /*
2201         * This check works well when we store new min/max freq attributes,
2202         * because new_policy is a copy of policy with one field updated.
2203         */
2204         if (new_policy->min > new_policy->max)
2205                 return -EINVAL;
2206
2207         /* verify the cpu speed can be set within this limit */
2208         ret = cpufreq_driver->verify(new_policy);
2209         if (ret)
2210                 return ret;
2211
2212         /* adjust if necessary - all reasons */
2213         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2214                         CPUFREQ_ADJUST, new_policy);
2215
2216         /*
2217          * verify the cpu speed can be set within this limit, which might be
2218          * different to the first one
2219          */
2220         ret = cpufreq_driver->verify(new_policy);
2221         if (ret)
2222                 return ret;
2223
2224         /* notification of the new policy */
2225         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2226                         CPUFREQ_NOTIFY, new_policy);
2227
2228         scale_freq_capacity(new_policy, NULL);
2229
2230         policy->min = new_policy->min;
2231         policy->max = new_policy->max;
2232         trace_cpu_frequency_limits(policy->max, policy->min, policy->cpu);
2233
2234         pr_debug("new min and max freqs are %u - %u kHz\n",
2235                  policy->min, policy->max);
2236
2237         if (cpufreq_driver->setpolicy) {
2238                 policy->policy = new_policy->policy;
2239                 pr_debug("setting range\n");
2240                 return cpufreq_driver->setpolicy(new_policy);
2241         }
2242
2243         if (new_policy->governor == policy->governor)
2244                 goto out;
2245
2246         pr_debug("governor switch\n");
2247
2248         /* save old, working values */
2249         old_gov = policy->governor;
2250         /* end old governor */
2251         if (old_gov) {
2252                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2253                 if (ret) {
2254                         /* This can happen due to race with other operations */
2255                         pr_debug("%s: Failed to Stop Governor: %s (%d)\n",
2256                                  __func__, old_gov->name, ret);
2257                         return ret;
2258                 }
2259
2260                 up_write(&policy->rwsem);
2261                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2262                 down_write(&policy->rwsem);
2263
2264                 if (ret) {
2265                         pr_err("%s: Failed to Exit Governor: %s (%d)\n",
2266                                __func__, old_gov->name, ret);
2267                         return ret;
2268                 }
2269         }
2270
2271         /* start new governor */
2272         policy->governor = new_policy->governor;
2273         ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2274         if (!ret) {
2275                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
2276                 if (!ret)
2277                         goto out;
2278
2279                 up_write(&policy->rwsem);
2280                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2281                 down_write(&policy->rwsem);
2282         }
2283
2284         /* new governor failed, so re-start old one */
2285         pr_debug("starting governor %s failed\n", policy->governor->name);
2286         if (old_gov) {
2287                 policy->governor = old_gov;
2288                 if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))
2289                         policy->governor = NULL;
2290                 else
2291                         __cpufreq_governor(policy, CPUFREQ_GOV_START);
2292         }
2293
2294         return ret;
2295
2296  out:
2297         pr_debug("governor: change or update limits\n");
2298         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2299 }
2300
2301 /**
2302  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2303  *      @cpu: CPU which shall be re-evaluated
2304  *
2305  *      Useful for policy notifiers which have different necessities
2306  *      at different times.
2307  */
2308 int cpufreq_update_policy(unsigned int cpu)
2309 {
2310         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2311         struct cpufreq_policy new_policy;
2312         int ret;
2313
2314         if (!policy)
2315                 return -ENODEV;
2316
2317         down_write(&policy->rwsem);
2318
2319         pr_debug("updating policy for CPU %u\n", cpu);
2320         memcpy(&new_policy, policy, sizeof(*policy));
2321         new_policy.min = policy->user_policy.min;
2322         new_policy.max = policy->user_policy.max;
2323
2324         /*
2325          * BIOS might change freq behind our back
2326          * -> ask driver for current freq and notify governors about a change
2327          */
2328         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2329                 new_policy.cur = cpufreq_driver->get(cpu);
2330                 if (WARN_ON(!new_policy.cur)) {
2331                         ret = -EIO;
2332                         goto unlock;
2333                 }
2334
2335                 if (!policy->cur) {
2336                         pr_debug("Driver did not initialize current freq\n");
2337                         policy->cur = new_policy.cur;
2338                 } else {
2339                         if (policy->cur != new_policy.cur && has_target())
2340                                 cpufreq_out_of_sync(policy, new_policy.cur);
2341                 }
2342         }
2343
2344         ret = cpufreq_set_policy(policy, &new_policy);
2345
2346 unlock:
2347         up_write(&policy->rwsem);
2348
2349         cpufreq_cpu_put(policy);
2350         return ret;
2351 }
2352 EXPORT_SYMBOL(cpufreq_update_policy);
2353
2354 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2355                                         unsigned long action, void *hcpu)
2356 {
2357         unsigned int cpu = (unsigned long)hcpu;
2358
2359         switch (action & ~CPU_TASKS_FROZEN) {
2360         case CPU_ONLINE:
2361                 cpufreq_online(cpu);
2362                 break;
2363
2364         case CPU_DOWN_PREPARE:
2365                 cpufreq_offline_prepare(cpu);
2366                 break;
2367
2368         case CPU_POST_DEAD:
2369                 cpufreq_offline_finish(cpu);
2370                 break;
2371
2372         case CPU_DOWN_FAILED:
2373                 cpufreq_online(cpu);
2374                 break;
2375         }
2376         return NOTIFY_OK;
2377 }
2378
2379 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2380         .notifier_call = cpufreq_cpu_callback,
2381 };
2382
2383 /*********************************************************************
2384  *               BOOST                                               *
2385  *********************************************************************/
2386 static int cpufreq_boost_set_sw(int state)
2387 {
2388         struct cpufreq_frequency_table *freq_table;
2389         struct cpufreq_policy *policy;
2390         int ret = -EINVAL;
2391
2392         for_each_active_policy(policy) {
2393                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2394                 if (freq_table) {
2395                         ret = cpufreq_frequency_table_cpuinfo(policy,
2396                                                         freq_table);
2397                         if (ret) {
2398                                 pr_err("%s: Policy frequency update failed\n",
2399                                        __func__);
2400                                 break;
2401                         }
2402                         policy->user_policy.max = policy->max;
2403                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2404                 }
2405         }
2406
2407         return ret;
2408 }
2409
2410 int cpufreq_boost_trigger_state(int state)
2411 {
2412         unsigned long flags;
2413         int ret = 0;
2414
2415         if (cpufreq_driver->boost_enabled == state)
2416                 return 0;
2417
2418         write_lock_irqsave(&cpufreq_driver_lock, flags);
2419         cpufreq_driver->boost_enabled = state;
2420         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2421
2422         ret = cpufreq_driver->set_boost(state);
2423         if (ret) {
2424                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2425                 cpufreq_driver->boost_enabled = !state;
2426                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2427
2428                 pr_err("%s: Cannot %s BOOST\n",
2429                        __func__, state ? "enable" : "disable");
2430         }
2431
2432         return ret;
2433 }
2434
2435 int cpufreq_boost_supported(void)
2436 {
2437         if (likely(cpufreq_driver))
2438                 return cpufreq_driver->boost_supported;
2439
2440         return 0;
2441 }
2442 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2443
2444 static int create_boost_sysfs_file(void)
2445 {
2446         int ret;
2447
2448         if (!cpufreq_boost_supported())
2449                 return 0;
2450
2451         /*
2452          * Check if driver provides function to enable boost -
2453          * if not, use cpufreq_boost_set_sw as default
2454          */
2455         if (!cpufreq_driver->set_boost)
2456                 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2457
2458         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2459         if (ret)
2460                 pr_err("%s: cannot register global BOOST sysfs file\n",
2461                        __func__);
2462
2463         return ret;
2464 }
2465
2466 static void remove_boost_sysfs_file(void)
2467 {
2468         if (cpufreq_boost_supported())
2469                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2470 }
2471
2472 int cpufreq_enable_boost_support(void)
2473 {
2474         if (!cpufreq_driver)
2475                 return -EINVAL;
2476
2477         if (cpufreq_boost_supported())
2478                 return 0;
2479
2480         cpufreq_driver->boost_supported = true;
2481
2482         /* This will get removed on driver unregister */
2483         return create_boost_sysfs_file();
2484 }
2485 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2486
2487 int cpufreq_boost_enabled(void)
2488 {
2489         return cpufreq_driver->boost_enabled;
2490 }
2491 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2492
2493 /*********************************************************************
2494  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2495  *********************************************************************/
2496
2497 /**
2498  * cpufreq_register_driver - register a CPU Frequency driver
2499  * @driver_data: A struct cpufreq_driver containing the values#
2500  * submitted by the CPU Frequency driver.
2501  *
2502  * Registers a CPU Frequency driver to this core code. This code
2503  * returns zero on success, -EBUSY when another driver got here first
2504  * (and isn't unregistered in the meantime).
2505  *
2506  */
2507 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2508 {
2509         unsigned long flags;
2510         int ret;
2511
2512         if (cpufreq_disabled())
2513                 return -ENODEV;
2514
2515         if (!driver_data || !driver_data->verify || !driver_data->init ||
2516             !(driver_data->setpolicy || driver_data->target_index ||
2517                     driver_data->target) ||
2518              (driver_data->setpolicy && (driver_data->target_index ||
2519                     driver_data->target)) ||
2520              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2521                 return -EINVAL;
2522
2523         pr_debug("trying to register driver %s\n", driver_data->name);
2524
2525         /* Protect against concurrent CPU online/offline. */
2526         get_online_cpus();
2527
2528         write_lock_irqsave(&cpufreq_driver_lock, flags);
2529         if (cpufreq_driver) {
2530                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2531                 ret = -EEXIST;
2532                 goto out;
2533         }
2534         cpufreq_driver = driver_data;
2535         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2536
2537         if (driver_data->setpolicy)
2538                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2539
2540         ret = create_boost_sysfs_file();
2541         if (ret)
2542                 goto err_null_driver;
2543
2544         ret = subsys_interface_register(&cpufreq_interface);
2545         if (ret)
2546                 goto err_boost_unreg;
2547
2548         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2549             list_empty(&cpufreq_policy_list)) {
2550                 /* if all ->init() calls failed, unregister */
2551                 ret = -ENODEV;
2552                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2553                          driver_data->name);
2554                 goto err_if_unreg;
2555         }
2556
2557         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2558         pr_debug("driver %s up and running\n", driver_data->name);
2559
2560 out:
2561         put_online_cpus();
2562         return ret;
2563
2564 err_if_unreg:
2565         subsys_interface_unregister(&cpufreq_interface);
2566 err_boost_unreg:
2567         remove_boost_sysfs_file();
2568 err_null_driver:
2569         write_lock_irqsave(&cpufreq_driver_lock, flags);
2570         cpufreq_driver = NULL;
2571         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2572         goto out;
2573 }
2574 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2575
2576 /**
2577  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2578  *
2579  * Unregister the current CPUFreq driver. Only call this if you have
2580  * the right to do so, i.e. if you have succeeded in initialising before!
2581  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2582  * currently not initialised.
2583  */
2584 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2585 {
2586         unsigned long flags;
2587
2588         if (!cpufreq_driver || (driver != cpufreq_driver))
2589                 return -EINVAL;
2590
2591         pr_debug("unregistering driver %s\n", driver->name);
2592
2593         /* Protect against concurrent cpu hotplug */
2594         get_online_cpus();
2595         subsys_interface_unregister(&cpufreq_interface);
2596         remove_boost_sysfs_file();
2597         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2598
2599         write_lock_irqsave(&cpufreq_driver_lock, flags);
2600
2601         cpufreq_driver = NULL;
2602
2603         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2604         put_online_cpus();
2605
2606         return 0;
2607 }
2608 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2609
2610 /*
2611  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2612  * or mutexes when secondary CPUs are halted.
2613  */
2614 static struct syscore_ops cpufreq_syscore_ops = {
2615         .shutdown = cpufreq_suspend,
2616 };
2617
2618 struct kobject *cpufreq_global_kobject;
2619 EXPORT_SYMBOL(cpufreq_global_kobject);
2620
2621 static int __init cpufreq_core_init(void)
2622 {
2623         if (cpufreq_disabled())
2624                 return -ENODEV;
2625
2626         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2627         BUG_ON(!cpufreq_global_kobject);
2628
2629         register_syscore_ops(&cpufreq_syscore_ops);
2630
2631         return 0;
2632 }
2633 core_initcall(cpufreq_core_init);