drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <[email protected]>
   6  *            (C) 2013 Viresh Kumar <[email protected]>
   7  *
   8  *  Oct 2005 - Ashok Raj <[email protected]>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <[email protected]>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 /* This one keeps track of the previously set governor of a removed CPU */
  46 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  47
  48 /* Flag to suspend/resume CPUFreq governors */
  49 static bool cpufreq_suspended;
  50
  51 static inline bool has_target(void)
  52 {
  53         return cpufreq_driver->target_index || cpufreq_driver->target;
  54 }
  55
  56 /*
  57  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  58  * sections
  59  */
  60 static DECLARE_RWSEM(cpufreq_rwsem);
  61
  62 /* internal prototypes */
  63 static int __cpufreq_governor(struct cpufreq_policy *policy,
  64                 unsigned int event);
  65 static unsigned int __cpufreq_get(unsigned int cpu);
  66 static void handle_update(struct work_struct *work);
  67
  68 /**
  69  * Two notifier lists: the "policy" list is involved in the
  70  * validation process for a new CPU frequency policy; the
  71  * "transition" list for kernel code that needs to handle
  72  * changes to devices when the CPU clock speed changes.
  73  * The mutex locks both lists.
  74  */
  75 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  76 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  77
  78 static bool init_cpufreq_transition_notifier_list_called;
  79 static int __init init_cpufreq_transition_notifier_list(void)
  80 {
  81         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  82         init_cpufreq_transition_notifier_list_called = true;
  83         return 0;
  84 }
  85 pure_initcall(init_cpufreq_transition_notifier_list);
  86
  87 static int off __read_mostly;
  88 static int cpufreq_disabled(void)
  89 {
  90         return off;
  91 }
  92 void disable_cpufreq(void)
  93 {
  94         off = 1;
  95 }
  96 static LIST_HEAD(cpufreq_governor_list);
  97 static DEFINE_MUTEX(cpufreq_governor_mutex);
  98
  99 bool have_governor_per_policy(void)
 100 {
 101         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 102 }
 103 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 104
 105 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 106 {
 107         if (have_governor_per_policy())
 108                 return &policy->kobj;
 109         else
 110                 return cpufreq_global_kobject;
 111 }
 112 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 113
 114 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 115 {
 116         u64 idle_time;
 117         u64 cur_wall_time;
 118         u64 busy_time;
 119
 120         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 121
 122         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 128
 129         idle_time = cur_wall_time - busy_time;
 130         if (wall)
 131                 *wall = cputime_to_usecs(cur_wall_time);
 132
 133         return cputime_to_usecs(idle_time);
 134 }
 135
 136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 137 {
 138         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 139
 140         if (idle_time == -1ULL)
 141                 return get_cpu_idle_time_jiffy(cpu, wall);
 142         else if (!io_busy)
 143                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 144
 145         return idle_time;
 146 }
 147 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 148
 149 /*
 150  * This is a generic cpufreq init() routine which can be used by cpufreq
 151  * drivers of SMP systems. It will do following:
 152  * - validate & show freq table passed
 153  * - set policies transition latency
 154  * - policy->cpus with all possible CPUs
 155  */
 156 int cpufreq_generic_init(struct cpufreq_policy *policy,
 157                 struct cpufreq_frequency_table *table,
 158                 unsigned int transition_latency)
 159 {
 160         int ret;
 161
 162         ret = cpufreq_table_validate_and_show(policy, table);
 163         if (ret) {
 164                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 165                 return ret;
 166         }
 167
 168         policy->cpuinfo.transition_latency = transition_latency;
 169
 170         /*
 171          * The driver only supports the SMP configuartion where all processors
 172          * share the clock and voltage and clock.
 173          */
 174         cpumask_setall(policy->cpus);
 175
 176         return 0;
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 179
 180 unsigned int cpufreq_generic_get(unsigned int cpu)
 181 {
 182         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 183
 184         if (!policy || IS_ERR(policy->clk)) {
 185                 pr_err("%s: No %s associated to cpu: %d\n",
 186                        __func__, policy ? "clk" : "policy", cpu);
 187                 return 0;
 188         }
 189
 190         return clk_get_rate(policy->clk) / 1000;
 191 }
 192 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 193
 194 /* Only for cpufreq core internal use */
 195 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 196 {
 197         return per_cpu(cpufreq_cpu_data, cpu);
 198 }
 199
 200 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 201 {
 202         struct cpufreq_policy *policy = NULL;
 203         unsigned long flags;
 204
 205         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 206                 return NULL;
 207
 208         if (!down_read_trylock(&cpufreq_rwsem))
 209                 return NULL;
 210
 211         /* get the cpufreq driver */
 212         read_lock_irqsave(&cpufreq_driver_lock, flags);
 213
 214         if (cpufreq_driver) {
 215                 /* get the CPU */
 216                 policy = per_cpu(cpufreq_cpu_data, cpu);
 217                 if (policy)
 218                         kobject_get(&policy->kobj);
 219         }
 220
 221         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 222
 223         if (!policy)
 224                 up_read(&cpufreq_rwsem);
 225
 226         return policy;
 227 }
 228 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 229
 230 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 231 {
 232         if (cpufreq_disabled())
 233                 return;
 234
 235         kobject_put(&policy->kobj);
 236         up_read(&cpufreq_rwsem);
 237 }
 238 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 239
 240 /*********************************************************************
 241  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 242  *********************************************************************/
 243
 244 /**
 245  * adjust_jiffies - adjust the system "loops_per_jiffy"
 246  *
 247  * This function alters the system "loops_per_jiffy" for the clock
 248  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 249  * systems as each CPU might be scaled differently. So, use the arch
 250  * per-CPU loops_per_jiffy value wherever possible.
 251  */
 252 #ifndef CONFIG_SMP
 253 static unsigned long l_p_j_ref;
 254 static unsigned int l_p_j_ref_freq;
 255
 256 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 257 {
 258         if (ci->flags & CPUFREQ_CONST_LOOPS)
 259                 return;
 260
 261         if (!l_p_j_ref_freq) {
 262                 l_p_j_ref = loops_per_jiffy;
 263                 l_p_j_ref_freq = ci->old;
 264                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 265                          l_p_j_ref, l_p_j_ref_freq);
 266         }
 267         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 268                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 269                                                                 ci->new);
 270                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 271                          loops_per_jiffy, ci->new);
 272         }
 273 }
 274 #else
 275 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 276 {
 277         return;
 278 }
 279 #endif
 280
 281 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 282                 struct cpufreq_freqs *freqs, unsigned int state)
 283 {
 284         BUG_ON(irqs_disabled());
 285
 286         if (cpufreq_disabled())
 287                 return;
 288
 289         freqs->flags = cpufreq_driver->flags;
 290         pr_debug("notification %u of frequency transition to %u kHz\n",
 291                  state, freqs->new);
 292
 293         switch (state) {
 294
 295         case CPUFREQ_PRECHANGE:
 296                 /* detect if the driver reported a value as "old frequency"
 297                  * which is not equal to what the cpufreq core thinks is
 298                  * "old frequency".
 299                  */
 300                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 301                         if ((policy) && (policy->cpu == freqs->cpu) &&
 302                             (policy->cur) && (policy->cur != freqs->old)) {
 303                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 304                                          freqs->old, policy->cur);
 305                                 freqs->old = policy->cur;
 306                         }
 307                 }
 308                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 309                                 CPUFREQ_PRECHANGE, freqs);
 310                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 311                 break;
 312
 313         case CPUFREQ_POSTCHANGE:
 314                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 315                 pr_debug("FREQ: %lu - CPU: %lu\n",
 316                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 317                 trace_cpu_frequency(freqs->new, freqs->cpu);
 318                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 319                                 CPUFREQ_POSTCHANGE, freqs);
 320                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 321                         policy->cur = freqs->new;
 322                 break;
 323         }
 324 }
 325
 326 /**
 327  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 328  * on frequency transition.
 329  *
 330  * This function calls the transition notifiers and the "adjust_jiffies"
 331  * function. It is called twice on all CPU frequency changes that have
 332  * external effects.
 333  */
 334 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 335                 struct cpufreq_freqs *freqs, unsigned int state)
 336 {
 337         for_each_cpu(freqs->cpu, policy->cpus)
 338                 __cpufreq_notify_transition(policy, freqs, state);
 339 }
 340
 341 /* Do post notifications when there are chances that transition has failed */
 342 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 343                 struct cpufreq_freqs *freqs, int transition_failed)
 344 {
 345         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 346         if (!transition_failed)
 347                 return;
 348
 349         swap(freqs->old, freqs->new);
 350         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 351         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 352 }
 353
 354 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 355                 struct cpufreq_freqs *freqs)
 356 {
 357 wait:
 358         wait_event(policy->transition_wait, !policy->transition_ongoing);
 359
 360         spin_lock(&policy->transition_lock);
 361
 362         if (unlikely(policy->transition_ongoing)) {
 363                 spin_unlock(&policy->transition_lock);
 364                 goto wait;
 365         }
 366
 367         policy->transition_ongoing = true;
 368
 369         spin_unlock(&policy->transition_lock);
 370
 371         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 372 }
 373 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 374
 375 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 376                 struct cpufreq_freqs *freqs, int transition_failed)
 377 {
 378         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 379                 return;
 380
 381         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 382
 383         policy->transition_ongoing = false;
 384
 385         wake_up(&policy->transition_wait);
 386 }
 387 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 388
 389
 390 /*********************************************************************
 391  *                          SYSFS INTERFACE                          *
 392  *********************************************************************/
 393 static ssize_t show_boost(struct kobject *kobj,
 394                                  struct attribute *attr, char *buf)
 395 {
 396         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 397 }
 398
 399 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 400                                   const char *buf, size_t count)
 401 {
 402         int ret, enable;
 403
 404         ret = sscanf(buf, "%d", &enable);
 405         if (ret != 1 || enable < 0 || enable > 1)
 406                 return -EINVAL;
 407
 408         if (cpufreq_boost_trigger_state(enable)) {
 409                 pr_err("%s: Cannot %s BOOST!\n",
 410                        __func__, enable ? "enable" : "disable");
 411                 return -EINVAL;
 412         }
 413
 414         pr_debug("%s: cpufreq BOOST %s\n",
 415                  __func__, enable ? "enabled" : "disabled");
 416
 417         return count;
 418 }
 419 define_one_global_rw(boost);
 420
 421 static struct cpufreq_governor *__find_governor(const char *str_governor)
 422 {
 423         struct cpufreq_governor *t;
 424
 425         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 426                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 427                         return t;
 428
 429         return NULL;
 430 }
 431
 432 /**
 433  * cpufreq_parse_governor - parse a governor string
 434  */
 435 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 436                                 struct cpufreq_governor **governor)
 437 {
 438         int err = -EINVAL;
 439
 440         if (!cpufreq_driver)
 441                 goto out;
 442
 443         if (cpufreq_driver->setpolicy) {
 444                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 445                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 446                         err = 0;
 447                 } else if (!strnicmp(str_governor, "powersave",
 448                                                 CPUFREQ_NAME_LEN)) {
 449                         *policy = CPUFREQ_POLICY_POWERSAVE;
 450                         err = 0;
 451                 }
 452         } else if (has_target()) {
 453                 struct cpufreq_governor *t;
 454
 455                 mutex_lock(&cpufreq_governor_mutex);
 456
 457                 t = __find_governor(str_governor);
 458
 459                 if (t == NULL) {
 460                         int ret;
 461
 462                         mutex_unlock(&cpufreq_governor_mutex);
 463                         ret = request_module("cpufreq_%s", str_governor);
 464                         mutex_lock(&cpufreq_governor_mutex);
 465
 466                         if (ret == 0)
 467                                 t = __find_governor(str_governor);
 468                 }
 469
 470                 if (t != NULL) {
 471                         *governor = t;
 472                         err = 0;
 473                 }
 474
 475                 mutex_unlock(&cpufreq_governor_mutex);
 476         }
 477 out:
 478         return err;
 479 }
 480
 481 /**
 482  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 483  * print out cpufreq information
 484  *
 485  * Write out information from cpufreq_driver->policy[cpu]; object must be
 486  * "unsigned int".
 487  */
 488
 489 #define show_one(file_name, object)                     \
 490 static ssize_t show_##file_name                         \
 491 (struct cpufreq_policy *policy, char *buf)              \
 492 {                                                       \
 493         return sprintf(buf, "%u\n", policy->object);    \
 494 }
 495
 496 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 497 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 498 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 499 show_one(scaling_min_freq, min);
 500 show_one(scaling_max_freq, max);
 501 show_one(scaling_cur_freq, cur);
 502
 503 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 504                                 struct cpufreq_policy *new_policy);
 505
 506 /**
 507  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 508  */
 509 #define store_one(file_name, object)                    \
 510 static ssize_t store_##file_name                                        \
 511 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 512 {                                                                       \
 513         int ret;                                                        \
 514         struct cpufreq_policy new_policy;                               \
 515                                                                         \
 516         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 517         if (ret)                                                        \
 518                 return -EINVAL;                                         \
 519                                                                         \
 520         ret = sscanf(buf, "%u", &new_policy.object);                    \
 521         if (ret != 1)                                                   \
 522                 return -EINVAL;                                         \
 523                                                                         \
 524         ret = cpufreq_set_policy(policy, &new_policy);          \
 525         policy->user_policy.object = policy->object;                    \
 526                                                                         \
 527         return ret ? ret : count;                                       \
 528 }
 529
 530 store_one(scaling_min_freq, min);
 531 store_one(scaling_max_freq, max);
 532
 533 /**
 534  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 535  */
 536 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 537                                         char *buf)
 538 {
 539         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 540         if (!cur_freq)
 541                 return sprintf(buf, "<unknown>");
 542         return sprintf(buf, "%u\n", cur_freq);
 543 }
 544
 545 /**
 546  * show_scaling_governor - show the current policy for the specified CPU
 547  */
 548 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 549 {
 550         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 551                 return sprintf(buf, "powersave\n");
 552         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 553                 return sprintf(buf, "performance\n");
 554         else if (policy->governor)
 555                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 556                                 policy->governor->name);
 557         return -EINVAL;
 558 }
 559
 560 /**
 561  * store_scaling_governor - store policy for the specified CPU
 562  */
 563 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 564                                         const char *buf, size_t count)
 565 {
 566         int ret;
 567         char    str_governor[16];
 568         struct cpufreq_policy new_policy;
 569
 570         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 571         if (ret)
 572                 return ret;
 573
 574         ret = sscanf(buf, "%15s", str_governor);
 575         if (ret != 1)
 576                 return -EINVAL;
 577
 578         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 579                                                 &new_policy.governor))
 580                 return -EINVAL;
 581
 582         ret = cpufreq_set_policy(policy, &new_policy);
 583
 584         policy->user_policy.policy = policy->policy;
 585         policy->user_policy.governor = policy->governor;
 586
 587         if (ret)
 588                 return ret;
 589         else
 590                 return count;
 591 }
 592
 593 /**
 594  * show_scaling_driver - show the cpufreq driver currently loaded
 595  */
 596 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 597 {
 598         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 599 }
 600
 601 /**
 602  * show_scaling_available_governors - show the available CPUfreq governors
 603  */
 604 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 605                                                 char *buf)
 606 {
 607         ssize_t i = 0;
 608         struct cpufreq_governor *t;
 609
 610         if (!has_target()) {
 611                 i += sprintf(buf, "performance powersave");
 612                 goto out;
 613         }
 614
 615         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 616                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 617                     - (CPUFREQ_NAME_LEN + 2)))
 618                         goto out;
 619                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 620         }
 621 out:
 622         i += sprintf(&buf[i], "\n");
 623         return i;
 624 }
 625
 626 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 627 {
 628         ssize_t i = 0;
 629         unsigned int cpu;
 630
 631         for_each_cpu(cpu, mask) {
 632                 if (i)
 633                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 634                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 635                 if (i >= (PAGE_SIZE - 5))
 636                         break;
 637         }
 638         i += sprintf(&buf[i], "\n");
 639         return i;
 640 }
 641 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 642
 643 /**
 644  * show_related_cpus - show the CPUs affected by each transition even if
 645  * hw coordination is in use
 646  */
 647 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 648 {
 649         return cpufreq_show_cpus(policy->related_cpus, buf);
 650 }
 651
 652 /**
 653  * show_affected_cpus - show the CPUs affected by each transition
 654  */
 655 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 656 {
 657         return cpufreq_show_cpus(policy->cpus, buf);
 658 }
 659
 660 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 661                                         const char *buf, size_t count)
 662 {
 663         unsigned int freq = 0;
 664         unsigned int ret;
 665
 666         if (!policy->governor || !policy->governor->store_setspeed)
 667                 return -EINVAL;
 668
 669         ret = sscanf(buf, "%u", &freq);
 670         if (ret != 1)
 671                 return -EINVAL;
 672
 673         policy->governor->store_setspeed(policy, freq);
 674
 675         return count;
 676 }
 677
 678 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 679 {
 680         if (!policy->governor || !policy->governor->show_setspeed)
 681                 return sprintf(buf, "<unsupported>\n");
 682
 683         return policy->governor->show_setspeed(policy, buf);
 684 }
 685
 686 /**
 687  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 688  */
 689 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 690 {
 691         unsigned int limit;
 692         int ret;
 693         if (cpufreq_driver->bios_limit) {
 694                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 695                 if (!ret)
 696                         return sprintf(buf, "%u\n", limit);
 697         }
 698         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 699 }
 700
 701 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 702 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 703 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 704 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 705 cpufreq_freq_attr_ro(scaling_available_governors);
 706 cpufreq_freq_attr_ro(scaling_driver);
 707 cpufreq_freq_attr_ro(scaling_cur_freq);
 708 cpufreq_freq_attr_ro(bios_limit);
 709 cpufreq_freq_attr_ro(related_cpus);
 710 cpufreq_freq_attr_ro(affected_cpus);
 711 cpufreq_freq_attr_rw(scaling_min_freq);
 712 cpufreq_freq_attr_rw(scaling_max_freq);
 713 cpufreq_freq_attr_rw(scaling_governor);
 714 cpufreq_freq_attr_rw(scaling_setspeed);
 715
 716 static struct attribute *default_attrs[] = {
 717         &cpuinfo_min_freq.attr,
 718         &cpuinfo_max_freq.attr,
 719         &cpuinfo_transition_latency.attr,
 720         &scaling_min_freq.attr,
 721         &scaling_max_freq.attr,
 722         &affected_cpus.attr,
 723         &related_cpus.attr,
 724         &scaling_governor.attr,
 725         &scaling_driver.attr,
 726         &scaling_available_governors.attr,
 727         &scaling_setspeed.attr,
 728         NULL
 729 };
 730
 731 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 732 #define to_attr(a) container_of(a, struct freq_attr, attr)
 733
 734 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 735 {
 736         struct cpufreq_policy *policy = to_policy(kobj);
 737         struct freq_attr *fattr = to_attr(attr);
 738         ssize_t ret;
 739
 740         if (!down_read_trylock(&cpufreq_rwsem))
 741                 return -EINVAL;
 742
 743         down_read(&policy->rwsem);
 744
 745         if (fattr->show)
 746                 ret = fattr->show(policy, buf);
 747         else
 748                 ret = -EIO;
 749
 750         up_read(&policy->rwsem);
 751         up_read(&cpufreq_rwsem);
 752
 753         return ret;
 754 }
 755
 756 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 757                      const char *buf, size_t count)
 758 {
 759         struct cpufreq_policy *policy = to_policy(kobj);
 760         struct freq_attr *fattr = to_attr(attr);
 761         ssize_t ret = -EINVAL;
 762
 763         get_online_cpus();
 764
 765         if (!cpu_online(policy->cpu))
 766                 goto unlock;
 767
 768         if (!down_read_trylock(&cpufreq_rwsem))
 769                 goto unlock;
 770
 771         down_write(&policy->rwsem);
 772
 773         if (fattr->store)
 774                 ret = fattr->store(policy, buf, count);
 775         else
 776                 ret = -EIO;
 777
 778         up_write(&policy->rwsem);
 779
 780         up_read(&cpufreq_rwsem);
 781 unlock:
 782         put_online_cpus();
 783
 784         return ret;
 785 }
 786
 787 static void cpufreq_sysfs_release(struct kobject *kobj)
 788 {
 789         struct cpufreq_policy *policy = to_policy(kobj);
 790         pr_debug("last reference is dropped\n");
 791         complete(&policy->kobj_unregister);
 792 }
 793
 794 static const struct sysfs_ops sysfs_ops = {
 795         .show   = show,
 796         .store  = store,
 797 };
 798
 799 static struct kobj_type ktype_cpufreq = {
 800         .sysfs_ops      = &sysfs_ops,
 801         .default_attrs  = default_attrs,
 802         .release        = cpufreq_sysfs_release,
 803 };
 804
 805 struct kobject *cpufreq_global_kobject;
 806 EXPORT_SYMBOL(cpufreq_global_kobject);
 807
 808 static int cpufreq_global_kobject_usage;
 809
 810 int cpufreq_get_global_kobject(void)
 811 {
 812         if (!cpufreq_global_kobject_usage++)
 813                 return kobject_add(cpufreq_global_kobject,
 814                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 815
 816         return 0;
 817 }
 818 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 819
 820 void cpufreq_put_global_kobject(void)
 821 {
 822         if (!--cpufreq_global_kobject_usage)
 823                 kobject_del(cpufreq_global_kobject);
 824 }
 825 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 826
 827 int cpufreq_sysfs_create_file(const struct attribute *attr)
 828 {
 829         int ret = cpufreq_get_global_kobject();
 830
 831         if (!ret) {
 832                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 833                 if (ret)
 834                         cpufreq_put_global_kobject();
 835         }
 836
 837         return ret;
 838 }
 839 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 840
 841 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 842 {
 843         sysfs_remove_file(cpufreq_global_kobject, attr);
 844         cpufreq_put_global_kobject();
 845 }
 846 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 847
 848 /* symlink affected CPUs */
 849 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 850 {
 851         unsigned int j;
 852         int ret = 0;
 853
 854         for_each_cpu(j, policy->cpus) {
 855                 struct device *cpu_dev;
 856
 857                 if (j == policy->cpu)
 858                         continue;
 859
 860                 pr_debug("Adding link for CPU: %u\n", j);
 861                 cpu_dev = get_cpu_device(j);
 862                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 863                                         "cpufreq");
 864                 if (ret)
 865                         break;
 866         }
 867         return ret;
 868 }
 869
 870 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 871                                      struct device *dev)
 872 {
 873         struct freq_attr **drv_attr;
 874         int ret = 0;
 875
 876         /* prepare interface data */
 877         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 878                                    &dev->kobj, "cpufreq");
 879         if (ret)
 880                 return ret;
 881
 882         /* set up files for this cpu device */
 883         drv_attr = cpufreq_driver->attr;
 884         while ((drv_attr) && (*drv_attr)) {
 885                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 886                 if (ret)
 887                         goto err_out_kobj_put;
 888                 drv_attr++;
 889         }
 890         if (cpufreq_driver->get) {
 891                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 892                 if (ret)
 893                         goto err_out_kobj_put;
 894         }
 895         if (has_target()) {
 896                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 897                 if (ret)
 898                         goto err_out_kobj_put;
 899         }
 900         if (cpufreq_driver->bios_limit) {
 901                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 902                 if (ret)
 903                         goto err_out_kobj_put;
 904         }
 905
 906         ret = cpufreq_add_dev_symlink(policy);
 907         if (ret)
 908                 goto err_out_kobj_put;
 909
 910         return ret;
 911
 912 err_out_kobj_put:
 913         kobject_put(&policy->kobj);
 914         wait_for_completion(&policy->kobj_unregister);
 915         return ret;
 916 }
 917
 918 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 919 {
 920         struct cpufreq_governor *gov = NULL;
 921         struct cpufreq_policy new_policy;
 922         int ret = 0;
 923
 924         memcpy(&new_policy, policy, sizeof(*policy));
 925
 926         /* Update governor of new_policy to the governor used before hotplug */
 927         gov = __find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 928         if (gov)
 929                 pr_debug("Restoring governor %s for cpu %d\n",
 930                                 policy->governor->name, policy->cpu);
 931         else
 932                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 933
 934         new_policy.governor = gov;
 935
 936         /* Use the default policy if its valid. */
 937         if (cpufreq_driver->setpolicy)
 938                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 939
 940         /* set default policy */
 941         ret = cpufreq_set_policy(policy, &new_policy);
 942         if (ret) {
 943                 pr_debug("setting policy failed\n");
 944                 if (cpufreq_driver->exit)
 945                         cpufreq_driver->exit(policy);
 946         }
 947 }
 948
 949 #ifdef CONFIG_HOTPLUG_CPU
 950 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 951                                   unsigned int cpu, struct device *dev)
 952 {
 953         int ret = 0;
 954         unsigned long flags;
 955
 956         if (has_target()) {
 957                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 958                 if (ret) {
 959                         pr_err("%s: Failed to stop governor\n", __func__);
 960                         return ret;
 961                 }
 962         }
 963
 964         down_write(&policy->rwsem);
 965
 966         write_lock_irqsave(&cpufreq_driver_lock, flags);
 967
 968         cpumask_set_cpu(cpu, policy->cpus);
 969         per_cpu(cpufreq_cpu_data, cpu) = policy;
 970         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 971
 972         up_write(&policy->rwsem);
 973
 974         if (has_target()) {
 975                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
 976                 if (!ret)
 977                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 978
 979                 if (ret) {
 980                         pr_err("%s: Failed to start governor\n", __func__);
 981                         return ret;
 982                 }
 983         }
 984
 985         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 986 }
 987 #endif
 988
 989 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 990 {
 991         struct cpufreq_policy *policy;
 992         unsigned long flags;
 993
 994         read_lock_irqsave(&cpufreq_driver_lock, flags);
 995
 996         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 997
 998         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 999
1000         policy->governor = NULL;
1001
1002         return policy;
1003 }
1004
1005 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1006 {
1007         struct cpufreq_policy *policy;
1008
1009         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1010         if (!policy)
1011                 return NULL;
1012
1013         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1014                 goto err_free_policy;
1015
1016         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1017                 goto err_free_cpumask;
1018
1019         INIT_LIST_HEAD(&policy->policy_list);
1020         init_rwsem(&policy->rwsem);
1021         spin_lock_init(&policy->transition_lock);
1022         init_waitqueue_head(&policy->transition_wait);
1023
1024         return policy;
1025
1026 err_free_cpumask:
1027         free_cpumask_var(policy->cpus);
1028 err_free_policy:
1029         kfree(policy);
1030
1031         return NULL;
1032 }
1033
1034 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1035 {
1036         struct kobject *kobj;
1037         struct completion *cmp;
1038
1039         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1040                         CPUFREQ_REMOVE_POLICY, policy);
1041
1042         down_read(&policy->rwsem);
1043         kobj = &policy->kobj;
1044         cmp = &policy->kobj_unregister;
1045         up_read(&policy->rwsem);
1046         kobject_put(kobj);
1047
1048         /*
1049          * We need to make sure that the underlying kobj is
1050          * actually not referenced anymore by anybody before we
1051          * proceed with unloading.
1052          */
1053         pr_debug("waiting for dropping of refcount\n");
1054         wait_for_completion(cmp);
1055         pr_debug("wait complete\n");
1056 }
1057
1058 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1059 {
1060         free_cpumask_var(policy->related_cpus);
1061         free_cpumask_var(policy->cpus);
1062         kfree(policy);
1063 }
1064
1065 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1066 {
1067         if (WARN_ON(cpu == policy->cpu))
1068                 return;
1069
1070         down_write(&policy->rwsem);
1071
1072         policy->last_cpu = policy->cpu;
1073         policy->cpu = cpu;
1074
1075         up_write(&policy->rwsem);
1076
1077         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1078                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1079 }
1080
1081 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1082 {
1083         unsigned int j, cpu = dev->id;
1084         int ret = -ENOMEM;
1085         struct cpufreq_policy *policy;
1086         unsigned long flags;
1087         bool recover_policy = cpufreq_suspended;
1088 #ifdef CONFIG_HOTPLUG_CPU
1089         struct cpufreq_policy *tpolicy;
1090 #endif
1091
1092         if (cpu_is_offline(cpu))
1093                 return 0;
1094
1095         pr_debug("adding CPU %u\n", cpu);
1096
1097 #ifdef CONFIG_SMP
1098         /* check whether a different CPU already registered this
1099          * CPU because it is in the same boat. */
1100         policy = cpufreq_cpu_get(cpu);
1101         if (unlikely(policy)) {
1102                 cpufreq_cpu_put(policy);
1103                 return 0;
1104         }
1105 #endif
1106
1107         if (!down_read_trylock(&cpufreq_rwsem))
1108                 return 0;
1109
1110 #ifdef CONFIG_HOTPLUG_CPU
1111         /* Check if this cpu was hot-unplugged earlier and has siblings */
1112         read_lock_irqsave(&cpufreq_driver_lock, flags);
1113         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1114                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1115                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1116                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1117                         up_read(&cpufreq_rwsem);
1118                         return ret;
1119                 }
1120         }
1121         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1122 #endif
1123
1124         /*
1125          * Restore the saved policy when doing light-weight init and fall back
1126          * to the full init if that fails.
1127          */
1128         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1129         if (!policy) {
1130                 recover_policy = false;
1131                 policy = cpufreq_policy_alloc();
1132                 if (!policy)
1133                         goto nomem_out;
1134         }
1135
1136         /*
1137          * In the resume path, since we restore a saved policy, the assignment
1138          * to policy->cpu is like an update of the existing policy, rather than
1139          * the creation of a brand new one. So we need to perform this update
1140          * by invoking update_policy_cpu().
1141          */
1142         if (recover_policy && cpu != policy->cpu)
1143                 update_policy_cpu(policy, cpu);
1144         else
1145                 policy->cpu = cpu;
1146
1147         cpumask_copy(policy->cpus, cpumask_of(cpu));
1148
1149         init_completion(&policy->kobj_unregister);
1150         INIT_WORK(&policy->update, handle_update);
1151
1152         /* call driver. From then on the cpufreq must be able
1153          * to accept all calls to ->verify and ->setpolicy for this CPU
1154          */
1155         ret = cpufreq_driver->init(policy);
1156         if (ret) {
1157                 pr_debug("initialization failed\n");
1158                 goto err_set_policy_cpu;
1159         }
1160
1161         /* related cpus should atleast have policy->cpus */
1162         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1163
1164         /*
1165          * affected cpus must always be the one, which are online. We aren't
1166          * managing offline cpus here.
1167          */
1168         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1169
1170         if (!recover_policy) {
1171                 policy->user_policy.min = policy->min;
1172                 policy->user_policy.max = policy->max;
1173         }
1174
1175         down_write(&policy->rwsem);
1176         write_lock_irqsave(&cpufreq_driver_lock, flags);
1177         for_each_cpu(j, policy->cpus)
1178                 per_cpu(cpufreq_cpu_data, j) = policy;
1179         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1180
1181         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1182                 policy->cur = cpufreq_driver->get(policy->cpu);
1183                 if (!policy->cur) {
1184                         pr_err("%s: ->get() failed\n", __func__);
1185                         goto err_get_freq;
1186                 }
1187         }
1188
1189         /*
1190          * Sometimes boot loaders set CPU frequency to a value outside of
1191          * frequency table present with cpufreq core. In such cases CPU might be
1192          * unstable if it has to run on that frequency for long duration of time
1193          * and so its better to set it to a frequency which is specified in
1194          * freq-table. This also makes cpufreq stats inconsistent as
1195          * cpufreq-stats would fail to register because current frequency of CPU
1196          * isn't found in freq-table.
1197          *
1198          * Because we don't want this change to effect boot process badly, we go
1199          * for the next freq which is >= policy->cur ('cur' must be set by now,
1200          * otherwise we will end up setting freq to lowest of the table as 'cur'
1201          * is initialized to zero).
1202          *
1203          * We are passing target-freq as "policy->cur - 1" otherwise
1204          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1205          * equal to target-freq.
1206          */
1207         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1208             && has_target()) {
1209                 /* Are we running at unknown frequency ? */
1210                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1211                 if (ret == -EINVAL) {
1212                         /* Warn user and fix it */
1213                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1214                                 __func__, policy->cpu, policy->cur);
1215                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1216                                 CPUFREQ_RELATION_L);
1217
1218                         /*
1219                          * Reaching here after boot in a few seconds may not
1220                          * mean that system will remain stable at "unknown"
1221                          * frequency for longer duration. Hence, a BUG_ON().
1222                          */
1223                         BUG_ON(ret);
1224                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1225                                 __func__, policy->cpu, policy->cur);
1226                 }
1227         }
1228
1229         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1230                                      CPUFREQ_START, policy);
1231
1232         if (!recover_policy) {
1233                 ret = cpufreq_add_dev_interface(policy, dev);
1234                 if (ret)
1235                         goto err_out_unregister;
1236                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1237                                 CPUFREQ_CREATE_POLICY, policy);
1238         }
1239
1240         write_lock_irqsave(&cpufreq_driver_lock, flags);
1241         list_add(&policy->policy_list, &cpufreq_policy_list);
1242         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1243
1244         cpufreq_init_policy(policy);
1245
1246         if (!recover_policy) {
1247                 policy->user_policy.policy = policy->policy;
1248                 policy->user_policy.governor = policy->governor;
1249         }
1250         up_write(&policy->rwsem);
1251
1252         kobject_uevent(&policy->kobj, KOBJ_ADD);
1253         up_read(&cpufreq_rwsem);
1254
1255         pr_debug("initialization complete\n");
1256
1257         return 0;
1258
1259 err_out_unregister:
1260 err_get_freq:
1261         write_lock_irqsave(&cpufreq_driver_lock, flags);
1262         for_each_cpu(j, policy->cpus)
1263                 per_cpu(cpufreq_cpu_data, j) = NULL;
1264         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1265
1266         if (cpufreq_driver->exit)
1267                 cpufreq_driver->exit(policy);
1268 err_set_policy_cpu:
1269         if (recover_policy) {
1270                 /* Do not leave stale fallback data behind. */
1271                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1272                 cpufreq_policy_put_kobj(policy);
1273         }
1274         cpufreq_policy_free(policy);
1275
1276 nomem_out:
1277         up_read(&cpufreq_rwsem);
1278
1279         return ret;
1280 }
1281
1282 /**
1283  * cpufreq_add_dev - add a CPU device
1284  *
1285  * Adds the cpufreq interface for a CPU device.
1286  *
1287  * The Oracle says: try running cpufreq registration/unregistration concurrently
1288  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1289  * mess up, but more thorough testing is needed. - Mathieu
1290  */
1291 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1292 {
1293         return __cpufreq_add_dev(dev, sif);
1294 }
1295
1296 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1297                                            unsigned int old_cpu)
1298 {
1299         struct device *cpu_dev;
1300         int ret;
1301
1302         /* first sibling now owns the new sysfs dir */
1303         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1304
1305         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1306         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1307         if (ret) {
1308                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1309
1310                 down_write(&policy->rwsem);
1311                 cpumask_set_cpu(old_cpu, policy->cpus);
1312                 up_write(&policy->rwsem);
1313
1314                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1315                                         "cpufreq");
1316
1317                 return -EINVAL;
1318         }
1319
1320         return cpu_dev->id;
1321 }
1322
1323 static int __cpufreq_remove_dev_prepare(struct device *dev,
1324                                         struct subsys_interface *sif)
1325 {
1326         unsigned int cpu = dev->id, cpus;
1327         int new_cpu, ret;
1328         unsigned long flags;
1329         struct cpufreq_policy *policy;
1330
1331         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1332
1333         write_lock_irqsave(&cpufreq_driver_lock, flags);
1334
1335         policy = per_cpu(cpufreq_cpu_data, cpu);
1336
1337         /* Save the policy somewhere when doing a light-weight tear-down */
1338         if (cpufreq_suspended)
1339                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1340
1341         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1342
1343         if (!policy) {
1344                 pr_debug("%s: No cpu_data found\n", __func__);
1345                 return -EINVAL;
1346         }
1347
1348         if (has_target()) {
1349                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1350                 if (ret) {
1351                         pr_err("%s: Failed to stop governor\n", __func__);
1352                         return ret;
1353                 }
1354         }
1355
1356         if (!cpufreq_driver->setpolicy)
1357                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1358                         policy->governor->name, CPUFREQ_NAME_LEN);
1359
1360         down_read(&policy->rwsem);
1361         cpus = cpumask_weight(policy->cpus);
1362         up_read(&policy->rwsem);
1363
1364         if (cpu != policy->cpu) {
1365                 sysfs_remove_link(&dev->kobj, "cpufreq");
1366         } else if (cpus > 1) {
1367                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1368                 if (new_cpu >= 0) {
1369                         update_policy_cpu(policy, new_cpu);
1370
1371                         if (!cpufreq_suspended)
1372                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1373                                          __func__, new_cpu, cpu);
1374                 }
1375         } else if (cpufreq_driver->stop_cpu && cpufreq_driver->setpolicy) {
1376                 cpufreq_driver->stop_cpu(policy);
1377         }
1378
1379         return 0;
1380 }
1381
1382 static int __cpufreq_remove_dev_finish(struct device *dev,
1383                                        struct subsys_interface *sif)
1384 {
1385         unsigned int cpu = dev->id, cpus;
1386         int ret;
1387         unsigned long flags;
1388         struct cpufreq_policy *policy;
1389
1390         read_lock_irqsave(&cpufreq_driver_lock, flags);
1391         policy = per_cpu(cpufreq_cpu_data, cpu);
1392         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1393
1394         if (!policy) {
1395                 pr_debug("%s: No cpu_data found\n", __func__);
1396                 return -EINVAL;
1397         }
1398
1399         down_write(&policy->rwsem);
1400         cpus = cpumask_weight(policy->cpus);
1401
1402         if (cpus > 1)
1403                 cpumask_clear_cpu(cpu, policy->cpus);
1404         up_write(&policy->rwsem);
1405
1406         /* If cpu is last user of policy, free policy */
1407         if (cpus == 1) {
1408                 if (has_target()) {
1409                         ret = __cpufreq_governor(policy,
1410                                         CPUFREQ_GOV_POLICY_EXIT);
1411                         if (ret) {
1412                                 pr_err("%s: Failed to exit governor\n",
1413                                        __func__);
1414                                 return ret;
1415                         }
1416                 }
1417
1418                 if (!cpufreq_suspended)
1419                         cpufreq_policy_put_kobj(policy);
1420
1421                 /*
1422                  * Perform the ->exit() even during light-weight tear-down,
1423                  * since this is a core component, and is essential for the
1424                  * subsequent light-weight ->init() to succeed.
1425                  */
1426                 if (cpufreq_driver->exit)
1427                         cpufreq_driver->exit(policy);
1428
1429                 /* Remove policy from list of active policies */
1430                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1431                 list_del(&policy->policy_list);
1432                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1433
1434                 if (!cpufreq_suspended)
1435                         cpufreq_policy_free(policy);
1436         } else if (has_target()) {
1437                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1438                 if (!ret)
1439                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1440
1441                 if (ret) {
1442                         pr_err("%s: Failed to start governor\n", __func__);
1443                         return ret;
1444                 }
1445         }
1446
1447         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1448         return 0;
1449 }
1450
1451 /**
1452  * cpufreq_remove_dev - remove a CPU device
1453  *
1454  * Removes the cpufreq interface for a CPU device.
1455  */
1456 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1457 {
1458         unsigned int cpu = dev->id;
1459         int ret;
1460
1461         if (cpu_is_offline(cpu))
1462                 return 0;
1463
1464         ret = __cpufreq_remove_dev_prepare(dev, sif);
1465
1466         if (!ret)
1467                 ret = __cpufreq_remove_dev_finish(dev, sif);
1468
1469         return ret;
1470 }
1471
1472 static void handle_update(struct work_struct *work)
1473 {
1474         struct cpufreq_policy *policy =
1475                 container_of(work, struct cpufreq_policy, update);
1476         unsigned int cpu = policy->cpu;
1477         pr_debug("handle_update for cpu %u called\n", cpu);
1478         cpufreq_update_policy(cpu);
1479 }
1480
1481 /**
1482  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1483  *      in deep trouble.
1484  *      @cpu: cpu number
1485  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1486  *      @new_freq: CPU frequency the CPU actually runs at
1487  *
1488  *      We adjust to current frequency first, and need to clean up later.
1489  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1490  */
1491 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1492                                 unsigned int new_freq)
1493 {
1494         struct cpufreq_policy *policy;
1495         struct cpufreq_freqs freqs;
1496         unsigned long flags;
1497
1498         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1499                  old_freq, new_freq);
1500
1501         freqs.old = old_freq;
1502         freqs.new = new_freq;
1503
1504         read_lock_irqsave(&cpufreq_driver_lock, flags);
1505         policy = per_cpu(cpufreq_cpu_data, cpu);
1506         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1507
1508         cpufreq_freq_transition_begin(policy, &freqs);
1509         cpufreq_freq_transition_end(policy, &freqs, 0);
1510 }
1511
1512 /**
1513  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1514  * @cpu: CPU number
1515  *
1516  * This is the last known freq, without actually getting it from the driver.
1517  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1518  */
1519 unsigned int cpufreq_quick_get(unsigned int cpu)
1520 {
1521         struct cpufreq_policy *policy;
1522         unsigned int ret_freq = 0;
1523
1524         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1525                 return cpufreq_driver->get(cpu);
1526
1527         policy = cpufreq_cpu_get(cpu);
1528         if (policy) {
1529                 ret_freq = policy->cur;
1530                 cpufreq_cpu_put(policy);
1531         }
1532
1533         return ret_freq;
1534 }
1535 EXPORT_SYMBOL(cpufreq_quick_get);
1536
1537 /**
1538  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1539  * @cpu: CPU number
1540  *
1541  * Just return the max possible frequency for a given CPU.
1542  */
1543 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1544 {
1545         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1546         unsigned int ret_freq = 0;
1547
1548         if (policy) {
1549                 ret_freq = policy->max;
1550                 cpufreq_cpu_put(policy);
1551         }
1552
1553         return ret_freq;
1554 }
1555 EXPORT_SYMBOL(cpufreq_quick_get_max);
1556
1557 static unsigned int __cpufreq_get(unsigned int cpu)
1558 {
1559         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1560         unsigned int ret_freq = 0;
1561
1562         if (!cpufreq_driver->get)
1563                 return ret_freq;
1564
1565         ret_freq = cpufreq_driver->get(cpu);
1566
1567         if (ret_freq && policy->cur &&
1568                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1569                 /* verify no discrepancy between actual and
1570                                         saved value exists */
1571                 if (unlikely(ret_freq != policy->cur)) {
1572                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1573                         schedule_work(&policy->update);
1574                 }
1575         }
1576
1577         return ret_freq;
1578 }
1579
1580 /**
1581  * cpufreq_get - get the current CPU frequency (in kHz)
1582  * @cpu: CPU number
1583  *
1584  * Get the CPU current (static) CPU frequency
1585  */
1586 unsigned int cpufreq_get(unsigned int cpu)
1587 {
1588         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1589         unsigned int ret_freq = 0;
1590
1591         if (policy) {
1592                 down_read(&policy->rwsem);
1593                 ret_freq = __cpufreq_get(cpu);
1594                 up_read(&policy->rwsem);
1595
1596                 cpufreq_cpu_put(policy);
1597         }
1598
1599         return ret_freq;
1600 }
1601 EXPORT_SYMBOL(cpufreq_get);
1602
1603 static struct subsys_interface cpufreq_interface = {
1604         .name           = "cpufreq",
1605         .subsys         = &cpu_subsys,
1606         .add_dev        = cpufreq_add_dev,
1607         .remove_dev     = cpufreq_remove_dev,
1608 };
1609
1610 /*
1611  * In case platform wants some specific frequency to be configured
1612  * during suspend..
1613  */
1614 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1615 {
1616         int ret;
1617
1618         if (!policy->suspend_freq) {
1619                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1620                 return -EINVAL;
1621         }
1622
1623         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1624                         policy->suspend_freq);
1625
1626         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1627                         CPUFREQ_RELATION_H);
1628         if (ret)
1629                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1630                                 __func__, policy->suspend_freq, ret);
1631
1632         return ret;
1633 }
1634 EXPORT_SYMBOL(cpufreq_generic_suspend);
1635
1636 /**
1637  * cpufreq_suspend() - Suspend CPUFreq governors
1638  *
1639  * Called during system wide Suspend/Hibernate cycles for suspending governors
1640  * as some platforms can't change frequency after this point in suspend cycle.
1641  * Because some of the devices (like: i2c, regulators, etc) they use for
1642  * changing frequency are suspended quickly after this point.
1643  */
1644 void cpufreq_suspend(void)
1645 {
1646         struct cpufreq_policy *policy;
1647
1648         if (!cpufreq_driver)
1649                 return;
1650
1651         if (!has_target())
1652                 return;
1653
1654         pr_debug("%s: Suspending Governors\n", __func__);
1655
1656         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1657                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1658                         pr_err("%s: Failed to stop governor for policy: %p\n",
1659                                 __func__, policy);
1660                 else if (cpufreq_driver->suspend
1661                     && cpufreq_driver->suspend(policy))
1662                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1663                                 policy);
1664         }
1665
1666         cpufreq_suspended = true;
1667 }
1668
1669 /**
1670  * cpufreq_resume() - Resume CPUFreq governors
1671  *
1672  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1673  * are suspended with cpufreq_suspend().
1674  */
1675 void cpufreq_resume(void)
1676 {
1677         struct cpufreq_policy *policy;
1678
1679         if (!cpufreq_driver)
1680                 return;
1681
1682         if (!has_target())
1683                 return;
1684
1685         pr_debug("%s: Resuming Governors\n", __func__);
1686
1687         cpufreq_suspended = false;
1688
1689         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1690                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1691                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1692                                 policy);
1693                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1694                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1695                         pr_err("%s: Failed to start governor for policy: %p\n",
1696                                 __func__, policy);
1697
1698                 /*
1699                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1700                  * policy in list. It will verify that the current freq is in
1701                  * sync with what we believe it to be.
1702                  */
1703                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1704                         schedule_work(&policy->update);
1705         }
1706 }
1707
1708 /**
1709  *      cpufreq_get_current_driver - return current driver's name
1710  *
1711  *      Return the name string of the currently loaded cpufreq driver
1712  *      or NULL, if none.
1713  */
1714 const char *cpufreq_get_current_driver(void)
1715 {
1716         if (cpufreq_driver)
1717                 return cpufreq_driver->name;
1718
1719         return NULL;
1720 }
1721 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1722
1723 /*********************************************************************
1724  *                     NOTIFIER LISTS INTERFACE                      *
1725  *********************************************************************/
1726
1727 /**
1728  *      cpufreq_register_notifier - register a driver with cpufreq
1729  *      @nb: notifier function to register
1730  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1731  *
1732  *      Add a driver to one of two lists: either a list of drivers that
1733  *      are notified about clock rate changes (once before and once after
1734  *      the transition), or a list of drivers that are notified about
1735  *      changes in cpufreq policy.
1736  *
1737  *      This function may sleep, and has the same return conditions as
1738  *      blocking_notifier_chain_register.
1739  */
1740 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1741 {
1742         int ret;
1743
1744         if (cpufreq_disabled())
1745                 return -EINVAL;
1746
1747         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1748
1749         switch (list) {
1750         case CPUFREQ_TRANSITION_NOTIFIER:
1751                 ret = srcu_notifier_chain_register(
1752                                 &cpufreq_transition_notifier_list, nb);
1753                 break;
1754         case CPUFREQ_POLICY_NOTIFIER:
1755                 ret = blocking_notifier_chain_register(
1756                                 &cpufreq_policy_notifier_list, nb);
1757                 break;
1758         default:
1759                 ret = -EINVAL;
1760         }
1761
1762         return ret;
1763 }
1764 EXPORT_SYMBOL(cpufreq_register_notifier);
1765
1766 /**
1767  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1768  *      @nb: notifier block to be unregistered
1769  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1770  *
1771  *      Remove a driver from the CPU frequency notifier list.
1772  *
1773  *      This function may sleep, and has the same return conditions as
1774  *      blocking_notifier_chain_unregister.
1775  */
1776 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1777 {
1778         int ret;
1779
1780         if (cpufreq_disabled())
1781                 return -EINVAL;
1782
1783         switch (list) {
1784         case CPUFREQ_TRANSITION_NOTIFIER:
1785                 ret = srcu_notifier_chain_unregister(
1786                                 &cpufreq_transition_notifier_list, nb);
1787                 break;
1788         case CPUFREQ_POLICY_NOTIFIER:
1789                 ret = blocking_notifier_chain_unregister(
1790                                 &cpufreq_policy_notifier_list, nb);
1791                 break;
1792         default:
1793                 ret = -EINVAL;
1794         }
1795
1796         return ret;
1797 }
1798 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1799
1800
1801 /*********************************************************************
1802  *                              GOVERNORS                            *
1803  *********************************************************************/
1804
1805 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1806                             unsigned int target_freq,
1807                             unsigned int relation)
1808 {
1809         int retval = -EINVAL;
1810         unsigned int old_target_freq = target_freq;
1811
1812         if (cpufreq_disabled())
1813                 return -ENODEV;
1814
1815         /* Make sure that target_freq is within supported range */
1816         if (target_freq > policy->max)
1817                 target_freq = policy->max;
1818         if (target_freq < policy->min)
1819                 target_freq = policy->min;
1820
1821         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1822                  policy->cpu, target_freq, relation, old_target_freq);
1823
1824         /*
1825          * This might look like a redundant call as we are checking it again
1826          * after finding index. But it is left intentionally for cases where
1827          * exactly same freq is called again and so we can save on few function
1828          * calls.
1829          */
1830         if (target_freq == policy->cur)
1831                 return 0;
1832
1833         if (cpufreq_driver->target)
1834                 retval = cpufreq_driver->target(policy, target_freq, relation);
1835         else if (cpufreq_driver->target_index) {
1836                 struct cpufreq_frequency_table *freq_table;
1837                 struct cpufreq_freqs freqs;
1838                 bool notify;
1839                 int index;
1840
1841                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1842                 if (unlikely(!freq_table)) {
1843                         pr_err("%s: Unable to find freq_table\n", __func__);
1844                         goto out;
1845                 }
1846
1847                 retval = cpufreq_frequency_table_target(policy, freq_table,
1848                                 target_freq, relation, &index);
1849                 if (unlikely(retval)) {
1850                         pr_err("%s: Unable to find matching freq\n", __func__);
1851                         goto out;
1852                 }
1853
1854                 if (freq_table[index].frequency == policy->cur) {
1855                         retval = 0;
1856                         goto out;
1857                 }
1858
1859                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1860
1861                 if (notify) {
1862                         freqs.old = policy->cur;
1863                         freqs.new = freq_table[index].frequency;
1864                         freqs.flags = 0;
1865
1866                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1867                                  __func__, policy->cpu, freqs.old, freqs.new);
1868
1869                         cpufreq_freq_transition_begin(policy, &freqs);
1870                 }
1871
1872                 retval = cpufreq_driver->target_index(policy, index);
1873                 if (retval)
1874                         pr_err("%s: Failed to change cpu frequency: %d\n",
1875                                __func__, retval);
1876
1877                 if (notify)
1878                         cpufreq_freq_transition_end(policy, &freqs, retval);
1879         }
1880
1881 out:
1882         return retval;
1883 }
1884 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1885
1886 int cpufreq_driver_target(struct cpufreq_policy *policy,
1887                           unsigned int target_freq,
1888                           unsigned int relation)
1889 {
1890         int ret = -EINVAL;
1891
1892         down_write(&policy->rwsem);
1893
1894         ret = __cpufreq_driver_target(policy, target_freq, relation);
1895
1896         up_write(&policy->rwsem);
1897
1898         return ret;
1899 }
1900 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1901
1902 /*
1903  * when "event" is CPUFREQ_GOV_LIMITS
1904  */
1905
1906 static int __cpufreq_governor(struct cpufreq_policy *policy,
1907                                         unsigned int event)
1908 {
1909         int ret;
1910
1911         /* Only must be defined when default governor is known to have latency
1912            restrictions, like e.g. conservative or ondemand.
1913            That this is the case is already ensured in Kconfig
1914         */
1915 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1916         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1917 #else
1918         struct cpufreq_governor *gov = NULL;
1919 #endif
1920
1921         /* Don't start any governor operations if we are entering suspend */
1922         if (cpufreq_suspended)
1923                 return 0;
1924
1925         if (policy->governor->max_transition_latency &&
1926             policy->cpuinfo.transition_latency >
1927             policy->governor->max_transition_latency) {
1928                 if (!gov)
1929                         return -EINVAL;
1930                 else {
1931                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
1932                                 policy->governor->name, gov->name);
1933                         policy->governor = gov;
1934                 }
1935         }
1936
1937         if (event == CPUFREQ_GOV_POLICY_INIT)
1938                 if (!try_module_get(policy->governor->owner))
1939                         return -EINVAL;
1940
1941         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1942                  policy->cpu, event);
1943
1944         mutex_lock(&cpufreq_governor_lock);
1945         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1946             || (!policy->governor_enabled
1947             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1948                 mutex_unlock(&cpufreq_governor_lock);
1949                 return -EBUSY;
1950         }
1951
1952         if (event == CPUFREQ_GOV_STOP)
1953                 policy->governor_enabled = false;
1954         else if (event == CPUFREQ_GOV_START)
1955                 policy->governor_enabled = true;
1956
1957         mutex_unlock(&cpufreq_governor_lock);
1958
1959         ret = policy->governor->governor(policy, event);
1960
1961         if (!ret) {
1962                 if (event == CPUFREQ_GOV_POLICY_INIT)
1963                         policy->governor->initialized++;
1964                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1965                         policy->governor->initialized--;
1966         } else {
1967                 /* Restore original values */
1968                 mutex_lock(&cpufreq_governor_lock);
1969                 if (event == CPUFREQ_GOV_STOP)
1970                         policy->governor_enabled = true;
1971                 else if (event == CPUFREQ_GOV_START)
1972                         policy->governor_enabled = false;
1973                 mutex_unlock(&cpufreq_governor_lock);
1974         }
1975
1976         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1977                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1978                 module_put(policy->governor->owner);
1979
1980         return ret;
1981 }
1982
1983 int cpufreq_register_governor(struct cpufreq_governor *governor)
1984 {
1985         int err;
1986
1987         if (!governor)
1988                 return -EINVAL;
1989
1990         if (cpufreq_disabled())
1991                 return -ENODEV;
1992
1993         mutex_lock(&cpufreq_governor_mutex);
1994
1995         governor->initialized = 0;
1996         err = -EBUSY;
1997         if (__find_governor(governor->name) == NULL) {
1998                 err = 0;
1999                 list_add(&governor->governor_list, &cpufreq_governor_list);
2000         }
2001
2002         mutex_unlock(&cpufreq_governor_mutex);
2003         return err;
2004 }
2005 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2006
2007 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2008 {
2009         int cpu;
2010
2011         if (!governor)
2012                 return;
2013
2014         if (cpufreq_disabled())
2015                 return;
2016
2017         for_each_present_cpu(cpu) {
2018                 if (cpu_online(cpu))
2019                         continue;
2020                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2021                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2022         }
2023
2024         mutex_lock(&cpufreq_governor_mutex);
2025         list_del(&governor->governor_list);
2026         mutex_unlock(&cpufreq_governor_mutex);
2027         return;
2028 }
2029 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2030
2031
2032 /*********************************************************************
2033  *                          POLICY INTERFACE                         *
2034  *********************************************************************/
2035
2036 /**
2037  * cpufreq_get_policy - get the current cpufreq_policy
2038  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2039  *      is written
2040  *
2041  * Reads the current cpufreq policy.
2042  */
2043 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2044 {
2045         struct cpufreq_policy *cpu_policy;
2046         if (!policy)
2047                 return -EINVAL;
2048
2049         cpu_policy = cpufreq_cpu_get(cpu);
2050         if (!cpu_policy)
2051                 return -EINVAL;
2052
2053         memcpy(policy, cpu_policy, sizeof(*policy));
2054
2055         cpufreq_cpu_put(cpu_policy);
2056         return 0;
2057 }
2058 EXPORT_SYMBOL(cpufreq_get_policy);
2059
2060 /*
2061  * policy : current policy.
2062  * new_policy: policy to be set.
2063  */
2064 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2065                                 struct cpufreq_policy *new_policy)
2066 {
2067         struct cpufreq_governor *old_gov;
2068         int ret;
2069
2070         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2071                  new_policy->cpu, new_policy->min, new_policy->max);
2072
2073         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2074
2075         if (new_policy->min > policy->max || new_policy->max < policy->min)
2076                 return -EINVAL;
2077
2078         /* verify the cpu speed can be set within this limit */
2079         ret = cpufreq_driver->verify(new_policy);
2080         if (ret)
2081                 return ret;
2082
2083         /* adjust if necessary - all reasons */
2084         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2085                         CPUFREQ_ADJUST, new_policy);
2086
2087         /* adjust if necessary - hardware incompatibility*/
2088         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2089                         CPUFREQ_INCOMPATIBLE, new_policy);
2090
2091         /*
2092          * verify the cpu speed can be set within this limit, which might be
2093          * different to the first one
2094          */
2095         ret = cpufreq_driver->verify(new_policy);
2096         if (ret)
2097                 return ret;
2098
2099         /* notification of the new policy */
2100         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2101                         CPUFREQ_NOTIFY, new_policy);
2102
2103         policy->min = new_policy->min;
2104         policy->max = new_policy->max;
2105
2106         pr_debug("new min and max freqs are %u - %u kHz\n",
2107                  policy->min, policy->max);
2108
2109         if (cpufreq_driver->setpolicy) {
2110                 policy->policy = new_policy->policy;
2111                 pr_debug("setting range\n");
2112                 return cpufreq_driver->setpolicy(new_policy);
2113         }
2114
2115         if (new_policy->governor == policy->governor)
2116                 goto out;
2117
2118         pr_debug("governor switch\n");
2119
2120         /* save old, working values */
2121         old_gov = policy->governor;
2122         /* end old governor */
2123         if (old_gov) {
2124                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2125                 up_write(&policy->rwsem);
2126                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2127                 down_write(&policy->rwsem);
2128         }
2129
2130         /* start new governor */
2131         policy->governor = new_policy->governor;
2132         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2133                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2134                         goto out;
2135
2136                 up_write(&policy->rwsem);
2137                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2138                 down_write(&policy->rwsem);
2139         }
2140
2141         /* new governor failed, so re-start old one */
2142         pr_debug("starting governor %s failed\n", policy->governor->name);
2143         if (old_gov) {
2144                 policy->governor = old_gov;
2145                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2146                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2147         }
2148
2149         return -EINVAL;
2150
2151  out:
2152         pr_debug("governor: change or update limits\n");
2153         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2154 }
2155
2156 /**
2157  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2158  *      @cpu: CPU which shall be re-evaluated
2159  *
2160  *      Useful for policy notifiers which have different necessities
2161  *      at different times.
2162  */
2163 int cpufreq_update_policy(unsigned int cpu)
2164 {
2165         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2166         struct cpufreq_policy new_policy;
2167         int ret;
2168
2169         if (!policy) {
2170                 ret = -ENODEV;
2171                 goto no_policy;
2172         }
2173
2174         down_write(&policy->rwsem);
2175
2176         pr_debug("updating policy for CPU %u\n", cpu);
2177         memcpy(&new_policy, policy, sizeof(*policy));
2178         new_policy.min = policy->user_policy.min;
2179         new_policy.max = policy->user_policy.max;
2180         new_policy.policy = policy->user_policy.policy;
2181         new_policy.governor = policy->user_policy.governor;
2182
2183         /*
2184          * BIOS might change freq behind our back
2185          * -> ask driver for current freq and notify governors about a change
2186          */
2187         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2188                 new_policy.cur = cpufreq_driver->get(cpu);
2189                 if (WARN_ON(!new_policy.cur)) {
2190                         ret = -EIO;
2191                         goto no_policy;
2192                 }
2193
2194                 if (!policy->cur) {
2195                         pr_debug("Driver did not initialize current freq\n");
2196                         policy->cur = new_policy.cur;
2197                 } else {
2198                         if (policy->cur != new_policy.cur && has_target())
2199                                 cpufreq_out_of_sync(cpu, policy->cur,
2200                                                                 new_policy.cur);
2201                 }
2202         }
2203
2204         ret = cpufreq_set_policy(policy, &new_policy);
2205
2206         up_write(&policy->rwsem);
2207
2208         cpufreq_cpu_put(policy);
2209 no_policy:
2210         return ret;
2211 }
2212 EXPORT_SYMBOL(cpufreq_update_policy);
2213
2214 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2215                                         unsigned long action, void *hcpu)
2216 {
2217         unsigned int cpu = (unsigned long)hcpu;
2218         struct device *dev;
2219
2220         dev = get_cpu_device(cpu);
2221         if (dev) {
2222                 switch (action & ~CPU_TASKS_FROZEN) {
2223                 case CPU_ONLINE:
2224                         __cpufreq_add_dev(dev, NULL);
2225                         break;
2226
2227                 case CPU_DOWN_PREPARE:
2228                         __cpufreq_remove_dev_prepare(dev, NULL);
2229                         break;
2230
2231                 case CPU_POST_DEAD:
2232                         __cpufreq_remove_dev_finish(dev, NULL);
2233                         break;
2234
2235                 case CPU_DOWN_FAILED:
2236                         __cpufreq_add_dev(dev, NULL);
2237                         break;
2238                 }
2239         }
2240         return NOTIFY_OK;
2241 }
2242
2243 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2244         .notifier_call = cpufreq_cpu_callback,
2245 };
2246
2247 /*********************************************************************
2248  *               BOOST                                               *
2249  *********************************************************************/
2250 static int cpufreq_boost_set_sw(int state)
2251 {
2252         struct cpufreq_frequency_table *freq_table;
2253         struct cpufreq_policy *policy;
2254         int ret = -EINVAL;
2255
2256         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2257                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2258                 if (freq_table) {
2259                         ret = cpufreq_frequency_table_cpuinfo(policy,
2260                                                         freq_table);
2261                         if (ret) {
2262                                 pr_err("%s: Policy frequency update failed\n",
2263                                        __func__);
2264                                 break;
2265                         }
2266                         policy->user_policy.max = policy->max;
2267                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2268                 }
2269         }
2270
2271         return ret;
2272 }
2273
2274 int cpufreq_boost_trigger_state(int state)
2275 {
2276         unsigned long flags;
2277         int ret = 0;
2278
2279         if (cpufreq_driver->boost_enabled == state)
2280                 return 0;
2281
2282         write_lock_irqsave(&cpufreq_driver_lock, flags);
2283         cpufreq_driver->boost_enabled = state;
2284         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2285
2286         ret = cpufreq_driver->set_boost(state);
2287         if (ret) {
2288                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2289                 cpufreq_driver->boost_enabled = !state;
2290                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2291
2292                 pr_err("%s: Cannot %s BOOST\n",
2293                        __func__, state ? "enable" : "disable");
2294         }
2295
2296         return ret;
2297 }
2298
2299 int cpufreq_boost_supported(void)
2300 {
2301         if (likely(cpufreq_driver))
2302                 return cpufreq_driver->boost_supported;
2303
2304         return 0;
2305 }
2306 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2307
2308 int cpufreq_boost_enabled(void)
2309 {
2310         return cpufreq_driver->boost_enabled;
2311 }
2312 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2313
2314 /*********************************************************************
2315  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2316  *********************************************************************/
2317
2318 /**
2319  * cpufreq_register_driver - register a CPU Frequency driver
2320  * @driver_data: A struct cpufreq_driver containing the values#
2321  * submitted by the CPU Frequency driver.
2322  *
2323  * Registers a CPU Frequency driver to this core code. This code
2324  * returns zero on success, -EBUSY when another driver got here first
2325  * (and isn't unregistered in the meantime).
2326  *
2327  */
2328 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2329 {
2330         unsigned long flags;
2331         int ret;
2332
2333         if (cpufreq_disabled())
2334                 return -ENODEV;
2335
2336         if (!driver_data || !driver_data->verify || !driver_data->init ||
2337             !(driver_data->setpolicy || driver_data->target_index ||
2338                     driver_data->target) ||
2339              (driver_data->setpolicy && (driver_data->target_index ||
2340                     driver_data->target)))
2341                 return -EINVAL;
2342
2343         pr_debug("trying to register driver %s\n", driver_data->name);
2344
2345         if (driver_data->setpolicy)
2346                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2347
2348         write_lock_irqsave(&cpufreq_driver_lock, flags);
2349         if (cpufreq_driver) {
2350                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2351                 return -EEXIST;
2352         }
2353         cpufreq_driver = driver_data;
2354         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2355
2356         if (cpufreq_boost_supported()) {
2357                 /*
2358                  * Check if driver provides function to enable boost -
2359                  * if not, use cpufreq_boost_set_sw as default
2360                  */
2361                 if (!cpufreq_driver->set_boost)
2362                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2363
2364                 ret = cpufreq_sysfs_create_file(&boost.attr);
2365                 if (ret) {
2366                         pr_err("%s: cannot register global BOOST sysfs file\n",
2367                                __func__);
2368                         goto err_null_driver;
2369                 }
2370         }
2371
2372         ret = subsys_interface_register(&cpufreq_interface);
2373         if (ret)
2374                 goto err_boost_unreg;
2375
2376         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2377                 int i;
2378                 ret = -ENODEV;
2379
2380                 /* check for at least one working CPU */
2381                 for (i = 0; i < nr_cpu_ids; i++)
2382                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2383                                 ret = 0;
2384                                 break;
2385                         }
2386
2387                 /* if all ->init() calls failed, unregister */
2388                 if (ret) {
2389                         pr_debug("no CPU initialized for driver %s\n",
2390                                  driver_data->name);
2391                         goto err_if_unreg;
2392                 }
2393         }
2394
2395         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2396         pr_debug("driver %s up and running\n", driver_data->name);
2397
2398         return 0;
2399 err_if_unreg:
2400         subsys_interface_unregister(&cpufreq_interface);
2401 err_boost_unreg:
2402         if (cpufreq_boost_supported())
2403                 cpufreq_sysfs_remove_file(&boost.attr);
2404 err_null_driver:
2405         write_lock_irqsave(&cpufreq_driver_lock, flags);
2406         cpufreq_driver = NULL;
2407         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2408         return ret;
2409 }
2410 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2411
2412 /**
2413  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2414  *
2415  * Unregister the current CPUFreq driver. Only call this if you have
2416  * the right to do so, i.e. if you have succeeded in initialising before!
2417  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2418  * currently not initialised.
2419  */
2420 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2421 {
2422         unsigned long flags;
2423
2424         if (!cpufreq_driver || (driver != cpufreq_driver))
2425                 return -EINVAL;
2426
2427         pr_debug("unregistering driver %s\n", driver->name);
2428
2429         subsys_interface_unregister(&cpufreq_interface);
2430         if (cpufreq_boost_supported())
2431                 cpufreq_sysfs_remove_file(&boost.attr);
2432
2433         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2434
2435         down_write(&cpufreq_rwsem);
2436         write_lock_irqsave(&cpufreq_driver_lock, flags);
2437
2438         cpufreq_driver = NULL;
2439
2440         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2441         up_write(&cpufreq_rwsem);
2442
2443         return 0;
2444 }
2445 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2446
2447 static int __init cpufreq_core_init(void)
2448 {
2449         if (cpufreq_disabled())
2450                 return -ENODEV;
2451
2452         cpufreq_global_kobject = kobject_create();
2453         BUG_ON(!cpufreq_global_kobject);
2454
2455         return 0;
2456 }
2457 core_initcall(cpufreq_core_init);