drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <[email protected]>
   6  *
   7  *  Oct 2005 - Ashok Raj <[email protected]>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <[email protected]>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31 #include <linux/syscore_ops.h>
  32
  33 #include <trace/events/power.h>
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  * - Lock should not be held across
  65  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  66  */
  67 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  68 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  69
  70 #define lock_policy_rwsem(mode, cpu)                                    \
  71 static int lock_policy_rwsem_##mode                                     \
  72 (int cpu)                                                               \
  73 {                                                                       \
  74         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  75         BUG_ON(policy_cpu == -1);                                       \
  76         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  77         if (unlikely(!cpu_online(cpu))) {                               \
  78                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  79                 return -1;                                              \
  80         }                                                               \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86
  87 lock_policy_rwsem(write, cpu);
  88
  89 static void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95
  96 static void unlock_policy_rwsem_write(int cpu)
  97 {
  98         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
  99         BUG_ON(policy_cpu == -1);
 100         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 101 }
 102
 103
 104 /* internal prototypes */
 105 static int __cpufreq_governor(struct cpufreq_policy *policy,
 106                 unsigned int event);
 107 static unsigned int __cpufreq_get(unsigned int cpu);
 108 static void handle_update(struct work_struct *work);
 109
 110 /**
 111  * Two notifier lists: the "policy" list is involved in the
 112  * validation process for a new CPU frequency policy; the
 113  * "transition" list for kernel code that needs to handle
 114  * changes to devices when the CPU clock speed changes.
 115  * The mutex locks both lists.
 116  */
 117 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 118 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 119
 120 static bool init_cpufreq_transition_notifier_list_called;
 121 static int __init init_cpufreq_transition_notifier_list(void)
 122 {
 123         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 124         init_cpufreq_transition_notifier_list_called = true;
 125         return 0;
 126 }
 127 pure_initcall(init_cpufreq_transition_notifier_list);
 128
 129 static LIST_HEAD(cpufreq_governor_list);
 130 static DEFINE_MUTEX(cpufreq_governor_mutex);
 131
 132 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 133 {
 134         struct cpufreq_policy *data;
 135         unsigned long flags;
 136
 137         if (cpu >= nr_cpu_ids)
 138                 goto err_out;
 139
 140         /* get the cpufreq driver */
 141         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 142
 143         if (!cpufreq_driver)
 144                 goto err_out_unlock;
 145
 146         if (!try_module_get(cpufreq_driver->owner))
 147                 goto err_out_unlock;
 148
 149
 150         /* get the CPU */
 151         data = per_cpu(cpufreq_cpu_data, cpu);
 152
 153         if (!data)
 154                 goto err_out_put_module;
 155
 156         if (!kobject_get(&data->kobj))
 157                 goto err_out_put_module;
 158
 159         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 160         return data;
 161
 162 err_out_put_module:
 163         module_put(cpufreq_driver->owner);
 164 err_out_unlock:
 165         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 166 err_out:
 167         return NULL;
 168 }
 169 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 170
 171
 172 void cpufreq_cpu_put(struct cpufreq_policy *data)
 173 {
 174         kobject_put(&data->kobj);
 175         module_put(cpufreq_driver->owner);
 176 }
 177 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 178
 179
 180 /*********************************************************************
 181  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 182  *********************************************************************/
 183
 184 /**
 185  * adjust_jiffies - adjust the system "loops_per_jiffy"
 186  *
 187  * This function alters the system "loops_per_jiffy" for the clock
 188  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 189  * systems as each CPU might be scaled differently. So, use the arch
 190  * per-CPU loops_per_jiffy value wherever possible.
 191  */
 192 #ifndef CONFIG_SMP
 193 static unsigned long l_p_j_ref;
 194 static unsigned int  l_p_j_ref_freq;
 195
 196 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 197 {
 198         if (ci->flags & CPUFREQ_CONST_LOOPS)
 199                 return;
 200
 201         if (!l_p_j_ref_freq) {
 202                 l_p_j_ref = loops_per_jiffy;
 203                 l_p_j_ref_freq = ci->old;
 204                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 205                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 206         }
 207         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 208             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 209             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 210                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 211                                                                 ci->new);
 212                 pr_debug("scaling loops_per_jiffy to %lu "
 213                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 214         }
 215 }
 216 #else
 217 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 218 {
 219         return;
 220 }
 221 #endif
 222
 223
 224 /**
 225  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 226  * on frequency transition.
 227  *
 228  * This function calls the transition notifiers and the "adjust_jiffies"
 229  * function. It is called twice on all CPU frequency changes that have
 230  * external effects.
 231  */
 232 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 233 {
 234         struct cpufreq_policy *policy;
 235
 236         BUG_ON(irqs_disabled());
 237
 238         freqs->flags = cpufreq_driver->flags;
 239         pr_debug("notification %u of frequency transition to %u kHz\n",
 240                 state, freqs->new);
 241
 242         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 243         switch (state) {
 244
 245         case CPUFREQ_PRECHANGE:
 246                 /* detect if the driver reported a value as "old frequency"
 247                  * which is not equal to what the cpufreq core thinks is
 248                  * "old frequency".
 249                  */
 250                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 251                         if ((policy) && (policy->cpu == freqs->cpu) &&
 252                             (policy->cur) && (policy->cur != freqs->old)) {
 253                                 pr_debug("Warning: CPU frequency is"
 254                                         " %u, cpufreq assumed %u kHz.\n",
 255                                         freqs->old, policy->cur);
 256                                 freqs->old = policy->cur;
 257                         }
 258                 }
 259                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 260                                 CPUFREQ_PRECHANGE, freqs);
 261                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 262                 break;
 263
 264         case CPUFREQ_POSTCHANGE:
 265                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 266                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 267                         (unsigned long)freqs->cpu);
 268                 trace_power_frequency(POWER_PSTATE, freqs->new, freqs->cpu);
 269                 trace_cpu_frequency(freqs->new, freqs->cpu);
 270                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 271                                 CPUFREQ_POSTCHANGE, freqs);
 272                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 273                         policy->cur = freqs->new;
 274                 break;
 275         }
 276 }
 277 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 278
 279
 280
 281 /*********************************************************************
 282  *                          SYSFS INTERFACE                          *
 283  *********************************************************************/
 284
 285 static struct cpufreq_governor *__find_governor(const char *str_governor)
 286 {
 287         struct cpufreq_governor *t;
 288
 289         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 290                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 291                         return t;
 292
 293         return NULL;
 294 }
 295
 296 /**
 297  * cpufreq_parse_governor - parse a governor string
 298  */
 299 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 300                                 struct cpufreq_governor **governor)
 301 {
 302         int err = -EINVAL;
 303
 304         if (!cpufreq_driver)
 305                 goto out;
 306
 307         if (cpufreq_driver->setpolicy) {
 308                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 309                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 310                         err = 0;
 311                 } else if (!strnicmp(str_governor, "powersave",
 312                                                 CPUFREQ_NAME_LEN)) {
 313                         *policy = CPUFREQ_POLICY_POWERSAVE;
 314                         err = 0;
 315                 }
 316         } else if (cpufreq_driver->target) {
 317                 struct cpufreq_governor *t;
 318
 319                 mutex_lock(&cpufreq_governor_mutex);
 320
 321                 t = __find_governor(str_governor);
 322
 323                 if (t == NULL) {
 324                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 325                                                                 str_governor);
 326
 327                         if (name) {
 328                                 int ret;
 329
 330                                 mutex_unlock(&cpufreq_governor_mutex);
 331                                 ret = request_module("%s", name);
 332                                 mutex_lock(&cpufreq_governor_mutex);
 333
 334                                 if (ret == 0)
 335                                         t = __find_governor(str_governor);
 336                         }
 337
 338                         kfree(name);
 339                 }
 340
 341                 if (t != NULL) {
 342                         *governor = t;
 343                         err = 0;
 344                 }
 345
 346                 mutex_unlock(&cpufreq_governor_mutex);
 347         }
 348 out:
 349         return err;
 350 }
 351
 352
 353 /**
 354  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 355  * print out cpufreq information
 356  *
 357  * Write out information from cpufreq_driver->policy[cpu]; object must be
 358  * "unsigned int".
 359  */
 360
 361 #define show_one(file_name, object)                     \
 362 static ssize_t show_##file_name                         \
 363 (struct cpufreq_policy *policy, char *buf)              \
 364 {                                                       \
 365         return sprintf(buf, "%u\n", policy->object);    \
 366 }
 367
 368 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 369 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 370 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 371 show_one(scaling_min_freq, min);
 372 show_one(scaling_max_freq, max);
 373 show_one(scaling_cur_freq, cur);
 374
 375 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 376                                 struct cpufreq_policy *policy);
 377
 378 /**
 379  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 380  */
 381 #define store_one(file_name, object)                    \
 382 static ssize_t store_##file_name                                        \
 383 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 384 {                                                                       \
 385         unsigned int ret = -EINVAL;                                     \
 386         struct cpufreq_policy new_policy;                               \
 387                                                                         \
 388         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 389         if (ret)                                                        \
 390                 return -EINVAL;                                         \
 391                                                                         \
 392         ret = sscanf(buf, "%u", &new_policy.object);                    \
 393         if (ret != 1)                                                   \
 394                 return -EINVAL;                                         \
 395                                                                         \
 396         ret = __cpufreq_set_policy(policy, &new_policy);                \
 397         policy->user_policy.object = policy->object;                    \
 398                                                                         \
 399         return ret ? ret : count;                                       \
 400 }
 401
 402 store_one(scaling_min_freq, min);
 403 store_one(scaling_max_freq, max);
 404
 405 /**
 406  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 407  */
 408 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 409                                         char *buf)
 410 {
 411         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 412         if (!cur_freq)
 413                 return sprintf(buf, "<unknown>");
 414         return sprintf(buf, "%u\n", cur_freq);
 415 }
 416
 417
 418 /**
 419  * show_scaling_governor - show the current policy for the specified CPU
 420  */
 421 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 422 {
 423         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 424                 return sprintf(buf, "powersave\n");
 425         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 426                 return sprintf(buf, "performance\n");
 427         else if (policy->governor)
 428                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 429                                 policy->governor->name);
 430         return -EINVAL;
 431 }
 432
 433
 434 /**
 435  * store_scaling_governor - store policy for the specified CPU
 436  */
 437 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 438                                         const char *buf, size_t count)
 439 {
 440         unsigned int ret = -EINVAL;
 441         char    str_governor[16];
 442         struct cpufreq_policy new_policy;
 443
 444         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 445         if (ret)
 446                 return ret;
 447
 448         ret = sscanf(buf, "%15s", str_governor);
 449         if (ret != 1)
 450                 return -EINVAL;
 451
 452         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 453                                                 &new_policy.governor))
 454                 return -EINVAL;
 455
 456         /* Do not use cpufreq_set_policy here or the user_policy.max
 457            will be wrongly overridden */
 458         ret = __cpufreq_set_policy(policy, &new_policy);
 459
 460         policy->user_policy.policy = policy->policy;
 461         policy->user_policy.governor = policy->governor;
 462
 463         if (ret)
 464                 return ret;
 465         else
 466                 return count;
 467 }
 468
 469 /**
 470  * show_scaling_driver - show the cpufreq driver currently loaded
 471  */
 472 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 473 {
 474         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 475 }
 476
 477 /**
 478  * show_scaling_available_governors - show the available CPUfreq governors
 479  */
 480 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 481                                                 char *buf)
 482 {
 483         ssize_t i = 0;
 484         struct cpufreq_governor *t;
 485
 486         if (!cpufreq_driver->target) {
 487                 i += sprintf(buf, "performance powersave");
 488                 goto out;
 489         }
 490
 491         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 492                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 493                     - (CPUFREQ_NAME_LEN + 2)))
 494                         goto out;
 495                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 496         }
 497 out:
 498         i += sprintf(&buf[i], "\n");
 499         return i;
 500 }
 501
 502 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 503 {
 504         ssize_t i = 0;
 505         unsigned int cpu;
 506
 507         for_each_cpu(cpu, mask) {
 508                 if (i)
 509                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 510                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 511                 if (i >= (PAGE_SIZE - 5))
 512                         break;
 513         }
 514         i += sprintf(&buf[i], "\n");
 515         return i;
 516 }
 517
 518 /**
 519  * show_related_cpus - show the CPUs affected by each transition even if
 520  * hw coordination is in use
 521  */
 522 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 523 {
 524         if (cpumask_empty(policy->related_cpus))
 525                 return show_cpus(policy->cpus, buf);
 526         return show_cpus(policy->related_cpus, buf);
 527 }
 528
 529 /**
 530  * show_affected_cpus - show the CPUs affected by each transition
 531  */
 532 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 533 {
 534         return show_cpus(policy->cpus, buf);
 535 }
 536
 537 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 538                                         const char *buf, size_t count)
 539 {
 540         unsigned int freq = 0;
 541         unsigned int ret;
 542
 543         if (!policy->governor || !policy->governor->store_setspeed)
 544                 return -EINVAL;
 545
 546         ret = sscanf(buf, "%u", &freq);
 547         if (ret != 1)
 548                 return -EINVAL;
 549
 550         policy->governor->store_setspeed(policy, freq);
 551
 552         return count;
 553 }
 554
 555 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 556 {
 557         if (!policy->governor || !policy->governor->show_setspeed)
 558                 return sprintf(buf, "<unsupported>\n");
 559
 560         return policy->governor->show_setspeed(policy, buf);
 561 }
 562
 563 /**
 564  * show_scaling_driver - show the current cpufreq HW/BIOS limitation
 565  */
 566 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 567 {
 568         unsigned int limit;
 569         int ret;
 570         if (cpufreq_driver->bios_limit) {
 571                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 572                 if (!ret)
 573                         return sprintf(buf, "%u\n", limit);
 574         }
 575         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 576 }
 577
 578 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 579 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 580 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 581 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 582 cpufreq_freq_attr_ro(scaling_available_governors);
 583 cpufreq_freq_attr_ro(scaling_driver);
 584 cpufreq_freq_attr_ro(scaling_cur_freq);
 585 cpufreq_freq_attr_ro(bios_limit);
 586 cpufreq_freq_attr_ro(related_cpus);
 587 cpufreq_freq_attr_ro(affected_cpus);
 588 cpufreq_freq_attr_rw(scaling_min_freq);
 589 cpufreq_freq_attr_rw(scaling_max_freq);
 590 cpufreq_freq_attr_rw(scaling_governor);
 591 cpufreq_freq_attr_rw(scaling_setspeed);
 592
 593 static struct attribute *default_attrs[] = {
 594         &cpuinfo_min_freq.attr,
 595         &cpuinfo_max_freq.attr,
 596         &cpuinfo_transition_latency.attr,
 597         &scaling_min_freq.attr,
 598         &scaling_max_freq.attr,
 599         &affected_cpus.attr,
 600         &related_cpus.attr,
 601         &scaling_governor.attr,
 602         &scaling_driver.attr,
 603         &scaling_available_governors.attr,
 604         &scaling_setspeed.attr,
 605         NULL
 606 };
 607
 608 struct kobject *cpufreq_global_kobject;
 609 EXPORT_SYMBOL(cpufreq_global_kobject);
 610
 611 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 612 #define to_attr(a) container_of(a, struct freq_attr, attr)
 613
 614 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 615 {
 616         struct cpufreq_policy *policy = to_policy(kobj);
 617         struct freq_attr *fattr = to_attr(attr);
 618         ssize_t ret = -EINVAL;
 619         policy = cpufreq_cpu_get(policy->cpu);
 620         if (!policy)
 621                 goto no_policy;
 622
 623         if (lock_policy_rwsem_read(policy->cpu) < 0)
 624                 goto fail;
 625
 626         if (fattr->show)
 627                 ret = fattr->show(policy, buf);
 628         else
 629                 ret = -EIO;
 630
 631         unlock_policy_rwsem_read(policy->cpu);
 632 fail:
 633         cpufreq_cpu_put(policy);
 634 no_policy:
 635         return ret;
 636 }
 637
 638 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 639                      const char *buf, size_t count)
 640 {
 641         struct cpufreq_policy *policy = to_policy(kobj);
 642         struct freq_attr *fattr = to_attr(attr);
 643         ssize_t ret = -EINVAL;
 644         policy = cpufreq_cpu_get(policy->cpu);
 645         if (!policy)
 646                 goto no_policy;
 647
 648         if (lock_policy_rwsem_write(policy->cpu) < 0)
 649                 goto fail;
 650
 651         if (fattr->store)
 652                 ret = fattr->store(policy, buf, count);
 653         else
 654                 ret = -EIO;
 655
 656         unlock_policy_rwsem_write(policy->cpu);
 657 fail:
 658         cpufreq_cpu_put(policy);
 659 no_policy:
 660         return ret;
 661 }
 662
 663 static void cpufreq_sysfs_release(struct kobject *kobj)
 664 {
 665         struct cpufreq_policy *policy = to_policy(kobj);
 666         pr_debug("last reference is dropped\n");
 667         complete(&policy->kobj_unregister);
 668 }
 669
 670 static const struct sysfs_ops sysfs_ops = {
 671         .show   = show,
 672         .store  = store,
 673 };
 674
 675 static struct kobj_type ktype_cpufreq = {
 676         .sysfs_ops      = &sysfs_ops,
 677         .default_attrs  = default_attrs,
 678         .release        = cpufreq_sysfs_release,
 679 };
 680
 681 /*
 682  * Returns:
 683  *   Negative: Failure
 684  *   0:        Success
 685  *   Positive: When we have a managed CPU and the sysfs got symlinked
 686  */
 687 static int cpufreq_add_dev_policy(unsigned int cpu,
 688                                   struct cpufreq_policy *policy,
 689                                   struct sys_device *sys_dev)
 690 {
 691         int ret = 0;
 692 #ifdef CONFIG_SMP
 693         unsigned long flags;
 694         unsigned int j;
 695 #ifdef CONFIG_HOTPLUG_CPU
 696         struct cpufreq_governor *gov;
 697
 698         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
 699         if (gov) {
 700                 policy->governor = gov;
 701                 pr_debug("Restoring governor %s for cpu %d\n",
 702                        policy->governor->name, cpu);
 703         }
 704 #endif
 705
 706         for_each_cpu(j, policy->cpus) {
 707                 struct cpufreq_policy *managed_policy;
 708
 709                 if (cpu == j)
 710                         continue;
 711
 712                 /* Check for existing affected CPUs.
 713                  * They may not be aware of it due to CPU Hotplug.
 714                  * cpufreq_cpu_put is called when the device is removed
 715                  * in __cpufreq_remove_dev()
 716                  */
 717                 managed_policy = cpufreq_cpu_get(j);
 718                 if (unlikely(managed_policy)) {
 719
 720                         /* Set proper policy_cpu */
 721                         unlock_policy_rwsem_write(cpu);
 722                         per_cpu(cpufreq_policy_cpu, cpu) = managed_policy->cpu;
 723
 724                         if (lock_policy_rwsem_write(cpu) < 0) {
 725                                 /* Should not go through policy unlock path */
 726                                 if (cpufreq_driver->exit)
 727                                         cpufreq_driver->exit(policy);
 728                                 cpufreq_cpu_put(managed_policy);
 729                                 return -EBUSY;
 730                         }
 731
 732                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 733                         cpumask_copy(managed_policy->cpus, policy->cpus);
 734                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 735                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 736
 737                         pr_debug("CPU already managed, adding link\n");
 738                         ret = sysfs_create_link(&sys_dev->kobj,
 739                                                 &managed_policy->kobj,
 740                                                 "cpufreq");
 741                         if (ret)
 742                                 cpufreq_cpu_put(managed_policy);
 743                         /*
 744                          * Success. We only needed to be added to the mask.
 745                          * Call driver->exit() because only the cpu parent of
 746                          * the kobj needed to call init().
 747                          */
 748                         if (cpufreq_driver->exit)
 749                                 cpufreq_driver->exit(policy);
 750
 751                         if (!ret)
 752                                 return 1;
 753                         else
 754                                 return ret;
 755                 }
 756         }
 757 #endif
 758         return ret;
 759 }
 760
 761
 762 /* symlink affected CPUs */
 763 static int cpufreq_add_dev_symlink(unsigned int cpu,
 764                                    struct cpufreq_policy *policy)
 765 {
 766         unsigned int j;
 767         int ret = 0;
 768
 769         for_each_cpu(j, policy->cpus) {
 770                 struct cpufreq_policy *managed_policy;
 771                 struct sys_device *cpu_sys_dev;
 772
 773                 if (j == cpu)
 774                         continue;
 775                 if (!cpu_online(j))
 776                         continue;
 777
 778                 pr_debug("CPU %u already managed, adding link\n", j);
 779                 managed_policy = cpufreq_cpu_get(cpu);
 780                 cpu_sys_dev = get_cpu_sysdev(j);
 781                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 782                                         "cpufreq");
 783                 if (ret) {
 784                         cpufreq_cpu_put(managed_policy);
 785                         return ret;
 786                 }
 787         }
 788         return ret;
 789 }
 790
 791 static int cpufreq_add_dev_interface(unsigned int cpu,
 792                                      struct cpufreq_policy *policy,
 793                                      struct sys_device *sys_dev)
 794 {
 795         struct cpufreq_policy new_policy;
 796         struct freq_attr **drv_attr;
 797         unsigned long flags;
 798         int ret = 0;
 799         unsigned int j;
 800
 801         /* prepare interface data */
 802         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 803                                    &sys_dev->kobj, "cpufreq");
 804         if (ret)
 805                 return ret;
 806
 807         /* set up files for this cpu device */
 808         drv_attr = cpufreq_driver->attr;
 809         while ((drv_attr) && (*drv_attr)) {
 810                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 811                 if (ret)
 812                         goto err_out_kobj_put;
 813                 drv_attr++;
 814         }
 815         if (cpufreq_driver->get) {
 816                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 817                 if (ret)
 818                         goto err_out_kobj_put;
 819         }
 820         if (cpufreq_driver->target) {
 821                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 822                 if (ret)
 823                         goto err_out_kobj_put;
 824         }
 825         if (cpufreq_driver->bios_limit) {
 826                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 827                 if (ret)
 828                         goto err_out_kobj_put;
 829         }
 830
 831         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 832         for_each_cpu(j, policy->cpus) {
 833                 if (!cpu_online(j))
 834                         continue;
 835                 per_cpu(cpufreq_cpu_data, j) = policy;
 836                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 837         }
 838         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 839
 840         ret = cpufreq_add_dev_symlink(cpu, policy);
 841         if (ret)
 842                 goto err_out_kobj_put;
 843
 844         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 845         /* assure that the starting sequence is run in __cpufreq_set_policy */
 846         policy->governor = NULL;
 847
 848         /* set default policy */
 849         ret = __cpufreq_set_policy(policy, &new_policy);
 850         policy->user_policy.policy = policy->policy;
 851         policy->user_policy.governor = policy->governor;
 852
 853         if (ret) {
 854                 pr_debug("setting policy failed\n");
 855                 if (cpufreq_driver->exit)
 856                         cpufreq_driver->exit(policy);
 857         }
 858         return ret;
 859
 860 err_out_kobj_put:
 861         kobject_put(&policy->kobj);
 862         wait_for_completion(&policy->kobj_unregister);
 863         return ret;
 864 }
 865
 866
 867 /**
 868  * cpufreq_add_dev - add a CPU device
 869  *
 870  * Adds the cpufreq interface for a CPU device.
 871  *
 872  * The Oracle says: try running cpufreq registration/unregistration concurrently
 873  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 874  * mess up, but more thorough testing is needed. - Mathieu
 875  */
 876 static int cpufreq_add_dev(struct sys_device *sys_dev)
 877 {
 878         unsigned int cpu = sys_dev->id;
 879         int ret = 0, found = 0;
 880         struct cpufreq_policy *policy;
 881         unsigned long flags;
 882         unsigned int j;
 883 #ifdef CONFIG_HOTPLUG_CPU
 884         int sibling;
 885 #endif
 886
 887         if (cpu_is_offline(cpu))
 888                 return 0;
 889
 890         pr_debug("adding CPU %u\n", cpu);
 891
 892 #ifdef CONFIG_SMP
 893         /* check whether a different CPU already registered this
 894          * CPU because it is in the same boat. */
 895         policy = cpufreq_cpu_get(cpu);
 896         if (unlikely(policy)) {
 897                 cpufreq_cpu_put(policy);
 898                 return 0;
 899         }
 900 #endif
 901
 902         if (!try_module_get(cpufreq_driver->owner)) {
 903                 ret = -EINVAL;
 904                 goto module_out;
 905         }
 906
 907         ret = -ENOMEM;
 908         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 909         if (!policy)
 910                 goto nomem_out;
 911
 912         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 913                 goto err_free_policy;
 914
 915         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 916                 goto err_free_cpumask;
 917
 918         policy->cpu = cpu;
 919         cpumask_copy(policy->cpus, cpumask_of(cpu));
 920
 921         /* Initially set CPU itself as the policy_cpu */
 922         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
 923         ret = (lock_policy_rwsem_write(cpu) < 0);
 924         WARN_ON(ret);
 925
 926         init_completion(&policy->kobj_unregister);
 927         INIT_WORK(&policy->update, handle_update);
 928
 929         /* Set governor before ->init, so that driver could check it */
 930 #ifdef CONFIG_HOTPLUG_CPU
 931         for_each_online_cpu(sibling) {
 932                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 933                 if (cp && cp->governor &&
 934                     (cpumask_test_cpu(cpu, cp->related_cpus))) {
 935                         policy->governor = cp->governor;
 936                         found = 1;
 937                         break;
 938                 }
 939         }
 940 #endif
 941         if (!found)
 942                 policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 943         /* call driver. From then on the cpufreq must be able
 944          * to accept all calls to ->verify and ->setpolicy for this CPU
 945          */
 946         ret = cpufreq_driver->init(policy);
 947         if (ret) {
 948                 pr_debug("initialization failed\n");
 949                 goto err_unlock_policy;
 950         }
 951         policy->user_policy.min = policy->min;
 952         policy->user_policy.max = policy->max;
 953
 954         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 955                                      CPUFREQ_START, policy);
 956
 957         ret = cpufreq_add_dev_policy(cpu, policy, sys_dev);
 958         if (ret) {
 959                 if (ret > 0)
 960                         /* This is a managed cpu, symlink created,
 961                            exit with 0 */
 962                         ret = 0;
 963                 goto err_unlock_policy;
 964         }
 965
 966         ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
 967         if (ret)
 968                 goto err_out_unregister;
 969
 970         unlock_policy_rwsem_write(cpu);
 971
 972         kobject_uevent(&policy->kobj, KOBJ_ADD);
 973         module_put(cpufreq_driver->owner);
 974         pr_debug("initialization complete\n");
 975
 976         return 0;
 977
 978
 979 err_out_unregister:
 980         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 981         for_each_cpu(j, policy->cpus)
 982                 per_cpu(cpufreq_cpu_data, j) = NULL;
 983         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 984
 985         kobject_put(&policy->kobj);
 986         wait_for_completion(&policy->kobj_unregister);
 987
 988 err_unlock_policy:
 989         unlock_policy_rwsem_write(cpu);
 990         free_cpumask_var(policy->related_cpus);
 991 err_free_cpumask:
 992         free_cpumask_var(policy->cpus);
 993 err_free_policy:
 994         kfree(policy);
 995 nomem_out:
 996         module_put(cpufreq_driver->owner);
 997 module_out:
 998         return ret;
 999 }
1000
1001
1002 /**
1003  * __cpufreq_remove_dev - remove a CPU device
1004  *
1005  * Removes the cpufreq interface for a CPU device.
1006  * Caller should already have policy_rwsem in write mode for this CPU.
1007  * This routine frees the rwsem before returning.
1008  */
1009 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1010 {
1011         unsigned int cpu = sys_dev->id;
1012         unsigned long flags;
1013         struct cpufreq_policy *data;
1014         struct kobject *kobj;
1015         struct completion *cmp;
1016 #ifdef CONFIG_SMP
1017         struct sys_device *cpu_sys_dev;
1018         unsigned int j;
1019 #endif
1020
1021         pr_debug("unregistering CPU %u\n", cpu);
1022
1023         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1024         data = per_cpu(cpufreq_cpu_data, cpu);
1025
1026         if (!data) {
1027                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1028                 unlock_policy_rwsem_write(cpu);
1029                 return -EINVAL;
1030         }
1031         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1032
1033
1034 #ifdef CONFIG_SMP
1035         /* if this isn't the CPU which is the parent of the kobj, we
1036          * only need to unlink, put and exit
1037          */
1038         if (unlikely(cpu != data->cpu)) {
1039                 pr_debug("removing link\n");
1040                 cpumask_clear_cpu(cpu, data->cpus);
1041                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1042                 kobj = &sys_dev->kobj;
1043                 cpufreq_cpu_put(data);
1044                 unlock_policy_rwsem_write(cpu);
1045                 sysfs_remove_link(kobj, "cpufreq");
1046                 return 0;
1047         }
1048 #endif
1049
1050 #ifdef CONFIG_SMP
1051
1052 #ifdef CONFIG_HOTPLUG_CPU
1053         strncpy(per_cpu(cpufreq_cpu_governor, cpu), data->governor->name,
1054                         CPUFREQ_NAME_LEN);
1055 #endif
1056
1057         /* if we have other CPUs still registered, we need to unlink them,
1058          * or else wait_for_completion below will lock up. Clean the
1059          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1060          * the sysfs links afterwards.
1061          */
1062         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1063                 for_each_cpu(j, data->cpus) {
1064                         if (j == cpu)
1065                                 continue;
1066                         per_cpu(cpufreq_cpu_data, j) = NULL;
1067                 }
1068         }
1069
1070         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1071
1072         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1073                 for_each_cpu(j, data->cpus) {
1074                         if (j == cpu)
1075                                 continue;
1076                         pr_debug("removing link for cpu %u\n", j);
1077 #ifdef CONFIG_HOTPLUG_CPU
1078                         strncpy(per_cpu(cpufreq_cpu_governor, j),
1079                                 data->governor->name, CPUFREQ_NAME_LEN);
1080 #endif
1081                         cpu_sys_dev = get_cpu_sysdev(j);
1082                         kobj = &cpu_sys_dev->kobj;
1083                         unlock_policy_rwsem_write(cpu);
1084                         sysfs_remove_link(kobj, "cpufreq");
1085                         lock_policy_rwsem_write(cpu);
1086                         cpufreq_cpu_put(data);
1087                 }
1088         }
1089 #else
1090         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1091 #endif
1092
1093         if (cpufreq_driver->target)
1094                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1095
1096         kobj = &data->kobj;
1097         cmp = &data->kobj_unregister;
1098         unlock_policy_rwsem_write(cpu);
1099         kobject_put(kobj);
1100
1101         /* we need to make sure that the underlying kobj is actually
1102          * not referenced anymore by anybody before we proceed with
1103          * unloading.
1104          */
1105         pr_debug("waiting for dropping of refcount\n");
1106         wait_for_completion(cmp);
1107         pr_debug("wait complete\n");
1108
1109         lock_policy_rwsem_write(cpu);
1110         if (cpufreq_driver->exit)
1111                 cpufreq_driver->exit(data);
1112         unlock_policy_rwsem_write(cpu);
1113
1114 #ifdef CONFIG_HOTPLUG_CPU
1115         /* when the CPU which is the parent of the kobj is hotplugged
1116          * offline, check for siblings, and create cpufreq sysfs interface
1117          * and symlinks
1118          */
1119         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1120                 /* first sibling now owns the new sysfs dir */
1121                 cpumask_clear_cpu(cpu, data->cpus);
1122                 cpufreq_add_dev(get_cpu_sysdev(cpumask_first(data->cpus)));
1123
1124                 /* finally remove our own symlink */
1125                 lock_policy_rwsem_write(cpu);
1126                 __cpufreq_remove_dev(sys_dev);
1127         }
1128 #endif
1129
1130         free_cpumask_var(data->related_cpus);
1131         free_cpumask_var(data->cpus);
1132         kfree(data);
1133
1134         return 0;
1135 }
1136
1137
1138 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1139 {
1140         unsigned int cpu = sys_dev->id;
1141         int retval;
1142
1143         if (cpu_is_offline(cpu))
1144                 return 0;
1145
1146         if (unlikely(lock_policy_rwsem_write(cpu)))
1147                 BUG();
1148
1149         retval = __cpufreq_remove_dev(sys_dev);
1150         return retval;
1151 }
1152
1153
1154 static void handle_update(struct work_struct *work)
1155 {
1156         struct cpufreq_policy *policy =
1157                 container_of(work, struct cpufreq_policy, update);
1158         unsigned int cpu = policy->cpu;
1159         pr_debug("handle_update for cpu %u called\n", cpu);
1160         cpufreq_update_policy(cpu);
1161 }
1162
1163 /**
1164  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1165  *      @cpu: cpu number
1166  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1167  *      @new_freq: CPU frequency the CPU actually runs at
1168  *
1169  *      We adjust to current frequency first, and need to clean up later.
1170  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1171  */
1172 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1173                                 unsigned int new_freq)
1174 {
1175         struct cpufreq_freqs freqs;
1176
1177         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1178                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1179
1180         freqs.cpu = cpu;
1181         freqs.old = old_freq;
1182         freqs.new = new_freq;
1183         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1184         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1185 }
1186
1187
1188 /**
1189  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1190  * @cpu: CPU number
1191  *
1192  * This is the last known freq, without actually getting it from the driver.
1193  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1194  */
1195 unsigned int cpufreq_quick_get(unsigned int cpu)
1196 {
1197         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1198         unsigned int ret_freq = 0;
1199
1200         if (policy) {
1201                 ret_freq = policy->cur;
1202                 cpufreq_cpu_put(policy);
1203         }
1204
1205         return ret_freq;
1206 }
1207 EXPORT_SYMBOL(cpufreq_quick_get);
1208
1209
1210 static unsigned int __cpufreq_get(unsigned int cpu)
1211 {
1212         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1213         unsigned int ret_freq = 0;
1214
1215         if (!cpufreq_driver->get)
1216                 return ret_freq;
1217
1218         ret_freq = cpufreq_driver->get(cpu);
1219
1220         if (ret_freq && policy->cur &&
1221                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1222                 /* verify no discrepancy between actual and
1223                                         saved value exists */
1224                 if (unlikely(ret_freq != policy->cur)) {
1225                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1226                         schedule_work(&policy->update);
1227                 }
1228         }
1229
1230         return ret_freq;
1231 }
1232
1233 /**
1234  * cpufreq_get - get the current CPU frequency (in kHz)
1235  * @cpu: CPU number
1236  *
1237  * Get the CPU current (static) CPU frequency
1238  */
1239 unsigned int cpufreq_get(unsigned int cpu)
1240 {
1241         unsigned int ret_freq = 0;
1242         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1243
1244         if (!policy)
1245                 goto out;
1246
1247         if (unlikely(lock_policy_rwsem_read(cpu)))
1248                 goto out_policy;
1249
1250         ret_freq = __cpufreq_get(cpu);
1251
1252         unlock_policy_rwsem_read(cpu);
1253
1254 out_policy:
1255         cpufreq_cpu_put(policy);
1256 out:
1257         return ret_freq;
1258 }
1259 EXPORT_SYMBOL(cpufreq_get);
1260
1261 static struct sysdev_driver cpufreq_sysdev_driver = {
1262         .add            = cpufreq_add_dev,
1263         .remove         = cpufreq_remove_dev,
1264 };
1265
1266
1267 /**
1268  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1269  *
1270  * This function is only executed for the boot processor.  The other CPUs
1271  * have been put offline by means of CPU hotplug.
1272  */
1273 static int cpufreq_bp_suspend(void)
1274 {
1275         int ret = 0;
1276
1277         int cpu = smp_processor_id();
1278         struct cpufreq_policy *cpu_policy;
1279
1280         pr_debug("suspending cpu %u\n", cpu);
1281
1282         /* If there's no policy for the boot CPU, we have nothing to do. */
1283         cpu_policy = cpufreq_cpu_get(cpu);
1284         if (!cpu_policy)
1285                 return 0;
1286
1287         if (cpufreq_driver->suspend) {
1288                 ret = cpufreq_driver->suspend(cpu_policy);
1289                 if (ret)
1290                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1291                                         "step on CPU %u\n", cpu_policy->cpu);
1292         }
1293
1294         cpufreq_cpu_put(cpu_policy);
1295         return ret;
1296 }
1297
1298 /**
1299  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1300  *
1301  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1302  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1303  *          restored. It will verify that the current freq is in sync with
1304  *          what we believe it to be. This is a bit later than when it
1305  *          should be, but nonethteless it's better than calling
1306  *          cpufreq_driver->get() here which might re-enable interrupts...
1307  *
1308  * This function is only executed for the boot CPU.  The other CPUs have not
1309  * been turned on yet.
1310  */
1311 static void cpufreq_bp_resume(void)
1312 {
1313         int ret = 0;
1314
1315         int cpu = smp_processor_id();
1316         struct cpufreq_policy *cpu_policy;
1317
1318         pr_debug("resuming cpu %u\n", cpu);
1319
1320         /* If there's no policy for the boot CPU, we have nothing to do. */
1321         cpu_policy = cpufreq_cpu_get(cpu);
1322         if (!cpu_policy)
1323                 return;
1324
1325         if (cpufreq_driver->resume) {
1326                 ret = cpufreq_driver->resume(cpu_policy);
1327                 if (ret) {
1328                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1329                                         "step on CPU %u\n", cpu_policy->cpu);
1330                         goto fail;
1331                 }
1332         }
1333
1334         schedule_work(&cpu_policy->update);
1335
1336 fail:
1337         cpufreq_cpu_put(cpu_policy);
1338 }
1339
1340 static struct syscore_ops cpufreq_syscore_ops = {
1341         .suspend        = cpufreq_bp_suspend,
1342         .resume         = cpufreq_bp_resume,
1343 };
1344
1345
1346 /*********************************************************************
1347  *                     NOTIFIER LISTS INTERFACE                      *
1348  *********************************************************************/
1349
1350 /**
1351  *      cpufreq_register_notifier - register a driver with cpufreq
1352  *      @nb: notifier function to register
1353  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1354  *
1355  *      Add a driver to one of two lists: either a list of drivers that
1356  *      are notified about clock rate changes (once before and once after
1357  *      the transition), or a list of drivers that are notified about
1358  *      changes in cpufreq policy.
1359  *
1360  *      This function may sleep, and has the same return conditions as
1361  *      blocking_notifier_chain_register.
1362  */
1363 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1364 {
1365         int ret;
1366
1367         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1368
1369         switch (list) {
1370         case CPUFREQ_TRANSITION_NOTIFIER:
1371                 ret = srcu_notifier_chain_register(
1372                                 &cpufreq_transition_notifier_list, nb);
1373                 break;
1374         case CPUFREQ_POLICY_NOTIFIER:
1375                 ret = blocking_notifier_chain_register(
1376                                 &cpufreq_policy_notifier_list, nb);
1377                 break;
1378         default:
1379                 ret = -EINVAL;
1380         }
1381
1382         return ret;
1383 }
1384 EXPORT_SYMBOL(cpufreq_register_notifier);
1385
1386
1387 /**
1388  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1389  *      @nb: notifier block to be unregistered
1390  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1391  *
1392  *      Remove a driver from the CPU frequency notifier list.
1393  *
1394  *      This function may sleep, and has the same return conditions as
1395  *      blocking_notifier_chain_unregister.
1396  */
1397 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1398 {
1399         int ret;
1400
1401         switch (list) {
1402         case CPUFREQ_TRANSITION_NOTIFIER:
1403                 ret = srcu_notifier_chain_unregister(
1404                                 &cpufreq_transition_notifier_list, nb);
1405                 break;
1406         case CPUFREQ_POLICY_NOTIFIER:
1407                 ret = blocking_notifier_chain_unregister(
1408                                 &cpufreq_policy_notifier_list, nb);
1409                 break;
1410         default:
1411                 ret = -EINVAL;
1412         }
1413
1414         return ret;
1415 }
1416 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1417
1418
1419 /*********************************************************************
1420  *                              GOVERNORS                            *
1421  *********************************************************************/
1422
1423
1424 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1425                             unsigned int target_freq,
1426                             unsigned int relation)
1427 {
1428         int retval = -EINVAL;
1429
1430         pr_debug("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1431                 target_freq, relation);
1432         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1433                 retval = cpufreq_driver->target(policy, target_freq, relation);
1434
1435         return retval;
1436 }
1437 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1438
1439 int cpufreq_driver_target(struct cpufreq_policy *policy,
1440                           unsigned int target_freq,
1441                           unsigned int relation)
1442 {
1443         int ret = -EINVAL;
1444
1445         policy = cpufreq_cpu_get(policy->cpu);
1446         if (!policy)
1447                 goto no_policy;
1448
1449         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1450                 goto fail;
1451
1452         ret = __cpufreq_driver_target(policy, target_freq, relation);
1453
1454         unlock_policy_rwsem_write(policy->cpu);
1455
1456 fail:
1457         cpufreq_cpu_put(policy);
1458 no_policy:
1459         return ret;
1460 }
1461 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1462
1463 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1464 {
1465         int ret = 0;
1466
1467         policy = cpufreq_cpu_get(policy->cpu);
1468         if (!policy)
1469                 return -EINVAL;
1470
1471         if (cpu_online(cpu) && cpufreq_driver->getavg)
1472                 ret = cpufreq_driver->getavg(policy, cpu);
1473
1474         cpufreq_cpu_put(policy);
1475         return ret;
1476 }
1477 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1478
1479 /*
1480  * when "event" is CPUFREQ_GOV_LIMITS
1481  */
1482
1483 static int __cpufreq_governor(struct cpufreq_policy *policy,
1484                                         unsigned int event)
1485 {
1486         int ret;
1487
1488         /* Only must be defined when default governor is known to have latency
1489            restrictions, like e.g. conservative or ondemand.
1490            That this is the case is already ensured in Kconfig
1491         */
1492 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1493         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1494 #else
1495         struct cpufreq_governor *gov = NULL;
1496 #endif
1497
1498         if (policy->governor->max_transition_latency &&
1499             policy->cpuinfo.transition_latency >
1500             policy->governor->max_transition_latency) {
1501                 if (!gov)
1502                         return -EINVAL;
1503                 else {
1504                         printk(KERN_WARNING "%s governor failed, too long"
1505                                " transition latency of HW, fallback"
1506                                " to %s governor\n",
1507                                policy->governor->name,
1508                                gov->name);
1509                         policy->governor = gov;
1510                 }
1511         }
1512
1513         if (!try_module_get(policy->governor->owner))
1514                 return -EINVAL;
1515
1516         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1517                                                 policy->cpu, event);
1518         ret = policy->governor->governor(policy, event);
1519
1520         /* we keep one module reference alive for
1521                         each CPU governed by this CPU */
1522         if ((event != CPUFREQ_GOV_START) || ret)
1523                 module_put(policy->governor->owner);
1524         if ((event == CPUFREQ_GOV_STOP) && !ret)
1525                 module_put(policy->governor->owner);
1526
1527         return ret;
1528 }
1529
1530
1531 int cpufreq_register_governor(struct cpufreq_governor *governor)
1532 {
1533         int err;
1534
1535         if (!governor)
1536                 return -EINVAL;
1537
1538         mutex_lock(&cpufreq_governor_mutex);
1539
1540         err = -EBUSY;
1541         if (__find_governor(governor->name) == NULL) {
1542                 err = 0;
1543                 list_add(&governor->governor_list, &cpufreq_governor_list);
1544         }
1545
1546         mutex_unlock(&cpufreq_governor_mutex);
1547         return err;
1548 }
1549 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1550
1551
1552 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1553 {
1554 #ifdef CONFIG_HOTPLUG_CPU
1555         int cpu;
1556 #endif
1557
1558         if (!governor)
1559                 return;
1560
1561 #ifdef CONFIG_HOTPLUG_CPU
1562         for_each_present_cpu(cpu) {
1563                 if (cpu_online(cpu))
1564                         continue;
1565                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1566                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1567         }
1568 #endif
1569
1570         mutex_lock(&cpufreq_governor_mutex);
1571         list_del(&governor->governor_list);
1572         mutex_unlock(&cpufreq_governor_mutex);
1573         return;
1574 }
1575 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1576
1577
1578
1579 /*********************************************************************
1580  *                          POLICY INTERFACE                         *
1581  *********************************************************************/
1582
1583 /**
1584  * cpufreq_get_policy - get the current cpufreq_policy
1585  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1586  *      is written
1587  *
1588  * Reads the current cpufreq policy.
1589  */
1590 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1591 {
1592         struct cpufreq_policy *cpu_policy;
1593         if (!policy)
1594                 return -EINVAL;
1595
1596         cpu_policy = cpufreq_cpu_get(cpu);
1597         if (!cpu_policy)
1598                 return -EINVAL;
1599
1600         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1601
1602         cpufreq_cpu_put(cpu_policy);
1603         return 0;
1604 }
1605 EXPORT_SYMBOL(cpufreq_get_policy);
1606
1607
1608 /*
1609  * data   : current policy.
1610  * policy : policy to be set.
1611  */
1612 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1613                                 struct cpufreq_policy *policy)
1614 {
1615         int ret = 0;
1616
1617         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1618                 policy->min, policy->max);
1619
1620         memcpy(&policy->cpuinfo, &data->cpuinfo,
1621                                 sizeof(struct cpufreq_cpuinfo));
1622
1623         if (policy->min > data->max || policy->max < data->min) {
1624                 ret = -EINVAL;
1625                 goto error_out;
1626         }
1627
1628         /* verify the cpu speed can be set within this limit */
1629         ret = cpufreq_driver->verify(policy);
1630         if (ret)
1631                 goto error_out;
1632
1633         /* adjust if necessary - all reasons */
1634         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1635                         CPUFREQ_ADJUST, policy);
1636
1637         /* adjust if necessary - hardware incompatibility*/
1638         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1639                         CPUFREQ_INCOMPATIBLE, policy);
1640
1641         /* verify the cpu speed can be set within this limit,
1642            which might be different to the first one */
1643         ret = cpufreq_driver->verify(policy);
1644         if (ret)
1645                 goto error_out;
1646
1647         /* notification of the new policy */
1648         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1649                         CPUFREQ_NOTIFY, policy);
1650
1651         data->min = policy->min;
1652         data->max = policy->max;
1653
1654         pr_debug("new min and max freqs are %u - %u kHz\n",
1655                                         data->min, data->max);
1656
1657         if (cpufreq_driver->setpolicy) {
1658                 data->policy = policy->policy;
1659                 pr_debug("setting range\n");
1660                 ret = cpufreq_driver->setpolicy(policy);
1661         } else {
1662                 if (policy->governor != data->governor) {
1663                         /* save old, working values */
1664                         struct cpufreq_governor *old_gov = data->governor;
1665
1666                         pr_debug("governor switch\n");
1667
1668                         /* end old governor */
1669                         if (data->governor)
1670                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1671
1672                         /* start new governor */
1673                         data->governor = policy->governor;
1674                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1675                                 /* new governor failed, so re-start old one */
1676                                 pr_debug("starting governor %s failed\n",
1677                                                         data->governor->name);
1678                                 if (old_gov) {
1679                                         data->governor = old_gov;
1680                                         __cpufreq_governor(data,
1681                                                            CPUFREQ_GOV_START);
1682                                 }
1683                                 ret = -EINVAL;
1684                                 goto error_out;
1685                         }
1686                         /* might be a policy change, too, so fall through */
1687                 }
1688                 pr_debug("governor: change or update limits\n");
1689                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1690         }
1691
1692 error_out:
1693         return ret;
1694 }
1695
1696 /**
1697  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1698  *      @cpu: CPU which shall be re-evaluated
1699  *
1700  *      Useful for policy notifiers which have different necessities
1701  *      at different times.
1702  */
1703 int cpufreq_update_policy(unsigned int cpu)
1704 {
1705         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1706         struct cpufreq_policy policy;
1707         int ret;
1708
1709         if (!data) {
1710                 ret = -ENODEV;
1711                 goto no_policy;
1712         }
1713
1714         if (unlikely(lock_policy_rwsem_write(cpu))) {
1715                 ret = -EINVAL;
1716                 goto fail;
1717         }
1718
1719         pr_debug("updating policy for CPU %u\n", cpu);
1720         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1721         policy.min = data->user_policy.min;
1722         policy.max = data->user_policy.max;
1723         policy.policy = data->user_policy.policy;
1724         policy.governor = data->user_policy.governor;
1725
1726         /* BIOS might change freq behind our back
1727           -> ask driver for current freq and notify governors about a change */
1728         if (cpufreq_driver->get) {
1729                 policy.cur = cpufreq_driver->get(cpu);
1730                 if (!data->cur) {
1731                         pr_debug("Driver did not initialize current freq");
1732                         data->cur = policy.cur;
1733                 } else {
1734                         if (data->cur != policy.cur)
1735                                 cpufreq_out_of_sync(cpu, data->cur,
1736                                                                 policy.cur);
1737                 }
1738         }
1739
1740         ret = __cpufreq_set_policy(data, &policy);
1741
1742         unlock_policy_rwsem_write(cpu);
1743
1744 fail:
1745         cpufreq_cpu_put(data);
1746 no_policy:
1747         return ret;
1748 }
1749 EXPORT_SYMBOL(cpufreq_update_policy);
1750
1751 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1752                                         unsigned long action, void *hcpu)
1753 {
1754         unsigned int cpu = (unsigned long)hcpu;
1755         struct sys_device *sys_dev;
1756
1757         sys_dev = get_cpu_sysdev(cpu);
1758         if (sys_dev) {
1759                 switch (action) {
1760                 case CPU_ONLINE:
1761                 case CPU_ONLINE_FROZEN:
1762                         cpufreq_add_dev(sys_dev);
1763                         break;
1764                 case CPU_DOWN_PREPARE:
1765                 case CPU_DOWN_PREPARE_FROZEN:
1766                         if (unlikely(lock_policy_rwsem_write(cpu)))
1767                                 BUG();
1768
1769                         __cpufreq_remove_dev(sys_dev);
1770                         break;
1771                 case CPU_DOWN_FAILED:
1772                 case CPU_DOWN_FAILED_FROZEN:
1773                         cpufreq_add_dev(sys_dev);
1774                         break;
1775                 }
1776         }
1777         return NOTIFY_OK;
1778 }
1779
1780 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1781     .notifier_call = cpufreq_cpu_callback,
1782 };
1783
1784 /*********************************************************************
1785  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1786  *********************************************************************/
1787
1788 /**
1789  * cpufreq_register_driver - register a CPU Frequency driver
1790  * @driver_data: A struct cpufreq_driver containing the values#
1791  * submitted by the CPU Frequency driver.
1792  *
1793  *   Registers a CPU Frequency driver to this core code. This code
1794  * returns zero on success, -EBUSY when another driver got here first
1795  * (and isn't unregistered in the meantime).
1796  *
1797  */
1798 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1799 {
1800         unsigned long flags;
1801         int ret;
1802
1803         if (!driver_data || !driver_data->verify || !driver_data->init ||
1804             ((!driver_data->setpolicy) && (!driver_data->target)))
1805                 return -EINVAL;
1806
1807         pr_debug("trying to register driver %s\n", driver_data->name);
1808
1809         if (driver_data->setpolicy)
1810                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1811
1812         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1813         if (cpufreq_driver) {
1814                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1815                 return -EBUSY;
1816         }
1817         cpufreq_driver = driver_data;
1818         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1819
1820         ret = sysdev_driver_register(&cpu_sysdev_class,
1821                                         &cpufreq_sysdev_driver);
1822         if (ret)
1823                 goto err_null_driver;
1824
1825         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1826                 int i;
1827                 ret = -ENODEV;
1828
1829                 /* check for at least one working CPU */
1830                 for (i = 0; i < nr_cpu_ids; i++)
1831                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1832                                 ret = 0;
1833                                 break;
1834                         }
1835
1836                 /* if all ->init() calls failed, unregister */
1837                 if (ret) {
1838                         pr_debug("no CPU initialized for driver %s\n",
1839                                                         driver_data->name);
1840                         goto err_sysdev_unreg;
1841                 }
1842         }
1843
1844         register_hotcpu_notifier(&cpufreq_cpu_notifier);
1845         pr_debug("driver %s up and running\n", driver_data->name);
1846
1847         return 0;
1848 err_sysdev_unreg:
1849         sysdev_driver_unregister(&cpu_sysdev_class,
1850                         &cpufreq_sysdev_driver);
1851 err_null_driver:
1852         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1853         cpufreq_driver = NULL;
1854         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1855         return ret;
1856 }
1857 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1858
1859
1860 /**
1861  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1862  *
1863  *    Unregister the current CPUFreq driver. Only call this if you have
1864  * the right to do so, i.e. if you have succeeded in initialising before!
1865  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1866  * currently not initialised.
1867  */
1868 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1869 {
1870         unsigned long flags;
1871
1872         if (!cpufreq_driver || (driver != cpufreq_driver))
1873                 return -EINVAL;
1874
1875         pr_debug("unregistering driver %s\n", driver->name);
1876
1877         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1878         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1879
1880         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1881         cpufreq_driver = NULL;
1882         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1883
1884         return 0;
1885 }
1886 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1887
1888 static int __init cpufreq_core_init(void)
1889 {
1890         int cpu;
1891
1892         for_each_possible_cpu(cpu) {
1893                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1894                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1895         }
1896
1897         cpufreq_global_kobject = kobject_create_and_add("cpufreq",
1898                                                 &cpu_sysdev_class.kset.kobj);
1899         BUG_ON(!cpufreq_global_kobject);
1900         register_syscore_ops(&cpufreq_syscore_ops);
1901
1902         return 0;
1903 }
1904 core_initcall(cpufreq_core_init);