SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static struct cpufreq_policy *cpufreq_cpu_data[NR_CPUS];
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static struct cpufreq_governor *cpufreq_cpu_governor[NR_CPUS];
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static bool init_cpufreq_transition_notifier_list_called;
 122 static int __init init_cpufreq_transition_notifier_list(void)
 123 {
 124         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 125         init_cpufreq_transition_notifier_list_called = true;
 126         return 0;
 127 }
 128 pure_initcall(init_cpufreq_transition_notifier_list);
 129
 130 static LIST_HEAD(cpufreq_governor_list);
 131 static DEFINE_MUTEX (cpufreq_governor_mutex);
 132
 133 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 134 {
 135         struct cpufreq_policy *data;
 136         unsigned long flags;
 137
 138         if (cpu >= NR_CPUS)
 139                 goto err_out;
 140
 141         /* get the cpufreq driver */
 142         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 143
 144         if (!cpufreq_driver)
 145                 goto err_out_unlock;
 146
 147         if (!try_module_get(cpufreq_driver->owner))
 148                 goto err_out_unlock;
 149
 150
 151         /* get the CPU */
 152         data = cpufreq_cpu_data[cpu];
 153
 154         if (!data)
 155                 goto err_out_put_module;
 156
 157         if (!kobject_get(&data->kobj))
 158                 goto err_out_put_module;
 159
 160         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 161         return data;
 162
 163 err_out_put_module:
 164         module_put(cpufreq_driver->owner);
 165 err_out_unlock:
 166         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 167 err_out:
 168         return NULL;
 169 }
 170 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 171
 172
 173 void cpufreq_cpu_put(struct cpufreq_policy *data)
 174 {
 175         kobject_put(&data->kobj);
 176         module_put(cpufreq_driver->owner);
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 179
 180
 181 /*********************************************************************
 182  *                     UNIFIED DEBUG HELPERS                         *
 183  *********************************************************************/
 184 #ifdef CONFIG_CPU_FREQ_DEBUG
 185
 186 /* what part(s) of the CPUfreq subsystem are debugged? */
 187 static unsigned int debug;
 188
 189 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 190  * set or modify this value.
 191  */
 192 static unsigned int debug_ratelimit = 1;
 193
 194 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 195  * loading of a cpufreq driver, temporarily disabled when a new policy
 196  * is set, and disabled upon cpufreq driver removal
 197  */
 198 static unsigned int disable_ratelimit = 1;
 199 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 200
 201 static void cpufreq_debug_enable_ratelimit(void)
 202 {
 203         unsigned long flags;
 204
 205         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 206         if (disable_ratelimit)
 207                 disable_ratelimit--;
 208         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 209 }
 210
 211 static void cpufreq_debug_disable_ratelimit(void)
 212 {
 213         unsigned long flags;
 214
 215         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 216         disable_ratelimit++;
 217         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 218 }
 219
 220 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 221                         const char *fmt, ...)
 222 {
 223         char s[256];
 224         va_list args;
 225         unsigned int len;
 226         unsigned long flags;
 227
 228         WARN_ON(!prefix);
 229         if (type & debug) {
 230                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 231                 if (!disable_ratelimit && debug_ratelimit
 232                                         && !printk_ratelimit()) {
 233                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 234                         return;
 235                 }
 236                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 237
 238                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 239
 240                 va_start(args, fmt);
 241                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 242                 va_end(args);
 243
 244                 printk(s);
 245
 246                 WARN_ON(len < 5);
 247         }
 248 }
 249 EXPORT_SYMBOL(cpufreq_debug_printk);
 250
 251
 252 module_param(debug, uint, 0644);
 253 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 254                         " 2 to debug drivers, and 4 to debug governors.");
 255
 256 module_param(debug_ratelimit, uint, 0644);
 257 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 258                                         " set to 0 to disable ratelimiting.");
 259
 260 #else /* !CONFIG_CPU_FREQ_DEBUG */
 261
 262 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 263 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 264
 265 #endif /* CONFIG_CPU_FREQ_DEBUG */
 266
 267
 268 /*********************************************************************
 269  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 270  *********************************************************************/
 271
 272 /**
 273  * adjust_jiffies - adjust the system "loops_per_jiffy"
 274  *
 275  * This function alters the system "loops_per_jiffy" for the clock
 276  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 277  * systems as each CPU might be scaled differently. So, use the arch
 278  * per-CPU loops_per_jiffy value wherever possible.
 279  */
 280 #ifndef CONFIG_SMP
 281 static unsigned long l_p_j_ref;
 282 static unsigned int  l_p_j_ref_freq;
 283
 284 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 285 {
 286         if (ci->flags & CPUFREQ_CONST_LOOPS)
 287                 return;
 288
 289         if (!l_p_j_ref_freq) {
 290                 l_p_j_ref = loops_per_jiffy;
 291                 l_p_j_ref_freq = ci->old;
 292                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 293                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 294         }
 295         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 296             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 297             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 298                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 299                                                                 ci->new);
 300                 dprintk("scaling loops_per_jiffy to %lu "
 301                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 302         }
 303 }
 304 #else
 305 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 306 {
 307         return;
 308 }
 309 #endif
 310
 311
 312 /**
 313  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 314  * on frequency transition.
 315  *
 316  * This function calls the transition notifiers and the "adjust_jiffies"
 317  * function. It is called twice on all CPU frequency changes that have
 318  * external effects.
 319  */
 320 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 321 {
 322         struct cpufreq_policy *policy;
 323
 324         BUG_ON(irqs_disabled());
 325
 326         freqs->flags = cpufreq_driver->flags;
 327         dprintk("notification %u of frequency transition to %u kHz\n",
 328                 state, freqs->new);
 329
 330         policy = cpufreq_cpu_data[freqs->cpu];
 331         switch (state) {
 332
 333         case CPUFREQ_PRECHANGE:
 334                 /* detect if the driver reported a value as "old frequency"
 335                  * which is not equal to what the cpufreq core thinks is
 336                  * "old frequency".
 337                  */
 338                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 339                         if ((policy) && (policy->cpu == freqs->cpu) &&
 340                             (policy->cur) && (policy->cur != freqs->old)) {
 341                                 dprintk("Warning: CPU frequency is"
 342                                         " %u, cpufreq assumed %u kHz.\n",
 343                                         freqs->old, policy->cur);
 344                                 freqs->old = policy->cur;
 345                         }
 346                 }
 347                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 348                                 CPUFREQ_PRECHANGE, freqs);
 349                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 350                 break;
 351
 352         case CPUFREQ_POSTCHANGE:
 353                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 354                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 355                                 CPUFREQ_POSTCHANGE, freqs);
 356                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 357                         policy->cur = freqs->new;
 358                 break;
 359         }
 360 }
 361 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 362
 363
 364
 365 /*********************************************************************
 366  *                          SYSFS INTERFACE                          *
 367  *********************************************************************/
 368
 369 static struct cpufreq_governor *__find_governor(const char *str_governor)
 370 {
 371         struct cpufreq_governor *t;
 372
 373         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 374                 if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 375                         return t;
 376
 377         return NULL;
 378 }
 379
 380 /**
 381  * cpufreq_parse_governor - parse a governor string
 382  */
 383 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 384                                 struct cpufreq_governor **governor)
 385 {
 386         int err = -EINVAL;
 387
 388         if (!cpufreq_driver)
 389                 goto out;
 390
 391         if (cpufreq_driver->setpolicy) {
 392                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 393                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 394                         err = 0;
 395                 } else if (!strnicmp(str_governor, "powersave",
 396                                                 CPUFREQ_NAME_LEN)) {
 397                         *policy = CPUFREQ_POLICY_POWERSAVE;
 398                         err = 0;
 399                 }
 400         } else if (cpufreq_driver->target) {
 401                 struct cpufreq_governor *t;
 402
 403                 mutex_lock(&cpufreq_governor_mutex);
 404
 405                 t = __find_governor(str_governor);
 406
 407                 if (t == NULL) {
 408                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 409                                                                 str_governor);
 410
 411                         if (name) {
 412                                 int ret;
 413
 414                                 mutex_unlock(&cpufreq_governor_mutex);
 415                                 ret = request_module(name);
 416                                 mutex_lock(&cpufreq_governor_mutex);
 417
 418                                 if (ret == 0)
 419                                         t = __find_governor(str_governor);
 420                         }
 421
 422                         kfree(name);
 423                 }
 424
 425                 if (t != NULL) {
 426                         *governor = t;
 427                         err = 0;
 428                 }
 429
 430                 mutex_unlock(&cpufreq_governor_mutex);
 431         }
 432   out:
 433         return err;
 434 }
 435
 436
 437 /* drivers/base/cpu.c */
 438 extern struct sysdev_class cpu_sysdev_class;
 439
 440
 441 /**
 442  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 443  * print out cpufreq information
 444  *
 445  * Write out information from cpufreq_driver->policy[cpu]; object must be
 446  * "unsigned int".
 447  */
 448
 449 #define show_one(file_name, object)                     \
 450 static ssize_t show_##file_name                         \
 451 (struct cpufreq_policy *policy, char *buf)              \
 452 {                                                       \
 453         return sprintf (buf, "%u\n", policy->object);   \
 454 }
 455
 456 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 457 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret = -EINVAL;                                     \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf (buf, "%u", &new_policy.object);                   \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq,min);
 490 store_one(scaling_max_freq,max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504
 505 /**
 506  * show_scaling_governor - show the current policy for the specified CPU
 507  */
 508 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 509 {
 510         if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 511                 return sprintf(buf, "powersave\n");
 512         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 513                 return sprintf(buf, "performance\n");
 514         else if (policy->governor)
 515                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 516         return -EINVAL;
 517 }
 518
 519
 520 /**
 521  * store_scaling_governor - store policy for the specified CPU
 522  */
 523 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 524                                         const char *buf, size_t count)
 525 {
 526         unsigned int ret = -EINVAL;
 527         char    str_governor[16];
 528         struct cpufreq_policy new_policy;
 529
 530         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 531         if (ret)
 532                 return ret;
 533
 534         ret = sscanf (buf, "%15s", str_governor);
 535         if (ret != 1)
 536                 return -EINVAL;
 537
 538         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 539                                                 &new_policy.governor))
 540                 return -EINVAL;
 541
 542         /* Do not use cpufreq_set_policy here or the user_policy.max
 543            will be wrongly overridden */
 544         ret = __cpufreq_set_policy(policy, &new_policy);
 545
 546         policy->user_policy.policy = policy->policy;
 547         policy->user_policy.governor = policy->governor;
 548
 549         if (ret)
 550                 return ret;
 551         else
 552                 return count;
 553 }
 554
 555 /**
 556  * show_scaling_driver - show the cpufreq driver currently loaded
 557  */
 558 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 559 {
 560         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 561 }
 562
 563 /**
 564  * show_scaling_available_governors - show the available CPUfreq governors
 565  */
 566 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 567                                                 char *buf)
 568 {
 569         ssize_t i = 0;
 570         struct cpufreq_governor *t;
 571
 572         if (!cpufreq_driver->target) {
 573                 i += sprintf(buf, "performance powersave");
 574                 goto out;
 575         }
 576
 577         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 578                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 579                         goto out;
 580                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581         }
 582 out:
 583         i += sprintf(&buf[i], "\n");
 584         return i;
 585 }
 586 /**
 587  * show_affected_cpus - show the CPUs affected by each transition
 588  */
 589 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu_mask(cpu, policy->cpus) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                     break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604
 605 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 606                                         const char *buf, size_t count)
 607 {
 608         unsigned int freq = 0;
 609         unsigned int ret;
 610
 611         if (!policy->governor->store_setspeed)
 612                 return -EINVAL;
 613
 614         ret = sscanf(buf, "%u", &freq);
 615         if (ret != 1)
 616                 return -EINVAL;
 617
 618         policy->governor->store_setspeed(policy, freq);
 619
 620         return count;
 621 }
 622
 623 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 624 {
 625         if (!policy->governor->show_setspeed)
 626                 return sprintf(buf, "<unsupported>\n");
 627
 628         return policy->governor->show_setspeed(policy, buf);
 629 }
 630
 631 #define define_one_ro(_name) \
 632 static struct freq_attr _name = \
 633 __ATTR(_name, 0444, show_##_name, NULL)
 634
 635 #define define_one_ro0400(_name) \
 636 static struct freq_attr _name = \
 637 __ATTR(_name, 0400, show_##_name, NULL)
 638
 639 #define define_one_rw(_name) \
 640 static struct freq_attr _name = \
 641 __ATTR(_name, 0644, show_##_name, store_##_name)
 642
 643 define_one_ro0400(cpuinfo_cur_freq);
 644 define_one_ro(cpuinfo_min_freq);
 645 define_one_ro(cpuinfo_max_freq);
 646 define_one_ro(scaling_available_governors);
 647 define_one_ro(scaling_driver);
 648 define_one_ro(scaling_cur_freq);
 649 define_one_ro(affected_cpus);
 650 define_one_rw(scaling_min_freq);
 651 define_one_rw(scaling_max_freq);
 652 define_one_rw(scaling_governor);
 653 define_one_rw(scaling_setspeed);
 654
 655 static struct attribute *default_attrs[] = {
 656         &cpuinfo_min_freq.attr,
 657         &cpuinfo_max_freq.attr,
 658         &scaling_min_freq.attr,
 659         &scaling_max_freq.attr,
 660         &affected_cpus.attr,
 661         &scaling_governor.attr,
 662         &scaling_driver.attr,
 663         &scaling_available_governors.attr,
 664         &scaling_setspeed.attr,
 665         NULL
 666 };
 667
 668 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 669 #define to_attr(a) container_of(a,struct freq_attr,attr)
 670
 671 static ssize_t show(struct kobject *kobj, struct attribute *attr ,char *buf)
 672 {
 673         struct cpufreq_policy *policy = to_policy(kobj);
 674         struct freq_attr *fattr = to_attr(attr);
 675         ssize_t ret = -EINVAL;
 676         policy = cpufreq_cpu_get(policy->cpu);
 677         if (!policy)
 678                 goto no_policy;
 679
 680         if (lock_policy_rwsem_read(policy->cpu) < 0)
 681                 goto fail;
 682
 683         if (fattr->show)
 684                 ret = fattr->show(policy, buf);
 685         else
 686                 ret = -EIO;
 687
 688         unlock_policy_rwsem_read(policy->cpu);
 689 fail:
 690         cpufreq_cpu_put(policy);
 691 no_policy:
 692         return ret;
 693 }
 694
 695 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 696                      const char *buf, size_t count)
 697 {
 698         struct cpufreq_policy *policy = to_policy(kobj);
 699         struct freq_attr *fattr = to_attr(attr);
 700         ssize_t ret = -EINVAL;
 701         policy = cpufreq_cpu_get(policy->cpu);
 702         if (!policy)
 703                 goto no_policy;
 704
 705         if (lock_policy_rwsem_write(policy->cpu) < 0)
 706                 goto fail;
 707
 708         if (fattr->store)
 709                 ret = fattr->store(policy, buf, count);
 710         else
 711                 ret = -EIO;
 712
 713         unlock_policy_rwsem_write(policy->cpu);
 714 fail:
 715         cpufreq_cpu_put(policy);
 716 no_policy:
 717         return ret;
 718 }
 719
 720 static void cpufreq_sysfs_release(struct kobject *kobj)
 721 {
 722         struct cpufreq_policy *policy = to_policy(kobj);
 723         dprintk("last reference is dropped\n");
 724         complete(&policy->kobj_unregister);
 725 }
 726
 727 static struct sysfs_ops sysfs_ops = {
 728         .show   = show,
 729         .store  = store,
 730 };
 731
 732 static struct kobj_type ktype_cpufreq = {
 733         .sysfs_ops      = &sysfs_ops,
 734         .default_attrs  = default_attrs,
 735         .release        = cpufreq_sysfs_release,
 736 };
 737
 738
 739 /**
 740  * cpufreq_add_dev - add a CPU device
 741  *
 742  * Adds the cpufreq interface for a CPU device.
 743  */
 744 static int cpufreq_add_dev(struct sys_device *sys_dev)
 745 {
 746         unsigned int cpu = sys_dev->id;
 747         int ret = 0;
 748         struct cpufreq_policy new_policy;
 749         struct cpufreq_policy *policy;
 750         struct freq_attr **drv_attr;
 751         struct sys_device *cpu_sys_dev;
 752         unsigned long flags;
 753         unsigned int j;
 754 #ifdef CONFIG_SMP
 755         struct cpufreq_policy *managed_policy;
 756 #endif
 757
 758         if (cpu_is_offline(cpu))
 759                 return 0;
 760
 761         cpufreq_debug_disable_ratelimit();
 762         dprintk("adding CPU %u\n", cpu);
 763
 764 #ifdef CONFIG_SMP
 765         /* check whether a different CPU already registered this
 766          * CPU because it is in the same boat. */
 767         policy = cpufreq_cpu_get(cpu);
 768         if (unlikely(policy)) {
 769                 cpufreq_cpu_put(policy);
 770                 cpufreq_debug_enable_ratelimit();
 771                 return 0;
 772         }
 773 #endif
 774
 775         if (!try_module_get(cpufreq_driver->owner)) {
 776                 ret = -EINVAL;
 777                 goto module_out;
 778         }
 779
 780         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 781         if (!policy) {
 782                 ret = -ENOMEM;
 783                 goto nomem_out;
 784         }
 785
 786         policy->cpu = cpu;
 787         policy->cpus = cpumask_of_cpu(cpu);
 788
 789         /* Initially set CPU itself as the policy_cpu */
 790         per_cpu(policy_cpu, cpu) = cpu;
 791         lock_policy_rwsem_write(cpu);
 792
 793         init_completion(&policy->kobj_unregister);
 794         INIT_WORK(&policy->update, handle_update);
 795
 796         /* Set governor before ->init, so that driver could check it */
 797         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 798         /* call driver. From then on the cpufreq must be able
 799          * to accept all calls to ->verify and ->setpolicy for this CPU
 800          */
 801         ret = cpufreq_driver->init(policy);
 802         if (ret) {
 803                 dprintk("initialization failed\n");
 804                 goto err_out;
 805         }
 806         policy->user_policy.min = policy->cpuinfo.min_freq;
 807         policy->user_policy.max = policy->cpuinfo.max_freq;
 808
 809 #ifdef CONFIG_SMP
 810
 811 #ifdef CONFIG_HOTPLUG_CPU
 812         if (cpufreq_cpu_governor[cpu]){
 813                 policy->governor = cpufreq_cpu_governor[cpu];
 814                 dprintk("Restoring governor %s for cpu %d\n",
 815                        policy->governor->name, cpu);
 816         }
 817 #endif
 818
 819         for_each_cpu_mask(j, policy->cpus) {
 820                 if (cpu == j)
 821                         continue;
 822
 823                 /* check for existing affected CPUs.  They may not be aware
 824                  * of it due to CPU Hotplug.
 825                  */
 826                 managed_policy = cpufreq_cpu_get(j);            // FIXME: Where is this released?  What about error paths?
 827                 if (unlikely(managed_policy)) {
 828
 829                         /* Set proper policy_cpu */
 830                         unlock_policy_rwsem_write(cpu);
 831                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 832
 833                         if (lock_policy_rwsem_write(cpu) < 0)
 834                                 goto err_out_driver_exit;
 835
 836                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 837                         managed_policy->cpus = policy->cpus;
 838                         cpufreq_cpu_data[cpu] = managed_policy;
 839                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 840
 841                         dprintk("CPU already managed, adding link\n");
 842                         ret = sysfs_create_link(&sys_dev->kobj,
 843                                                 &managed_policy->kobj,
 844                                                 "cpufreq");
 845                         if (ret)
 846                                 goto err_out_driver_exit;
 847
 848                         cpufreq_debug_enable_ratelimit();
 849                         ret = 0;
 850                         goto err_out_driver_exit; /* call driver->exit() */
 851                 }
 852         }
 853 #endif
 854         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 855
 856         /* prepare interface data */
 857         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 858                                    "cpufreq");
 859         if (ret)
 860                 goto err_out_driver_exit;
 861
 862         /* set up files for this cpu device */
 863         drv_attr = cpufreq_driver->attr;
 864         while ((drv_attr) && (*drv_attr)) {
 865                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 866                 if (ret)
 867                         goto err_out_driver_exit;
 868                 drv_attr++;
 869         }
 870         if (cpufreq_driver->get) {
 871                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 872                 if (ret)
 873                         goto err_out_driver_exit;
 874         }
 875         if (cpufreq_driver->target) {
 876                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 877                 if (ret)
 878                         goto err_out_driver_exit;
 879         }
 880
 881         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 882         for_each_cpu_mask(j, policy->cpus) {
 883                 cpufreq_cpu_data[j] = policy;
 884                 per_cpu(policy_cpu, j) = policy->cpu;
 885         }
 886         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 887
 888         /* symlink affected CPUs */
 889         for_each_cpu_mask(j, policy->cpus) {
 890                 if (j == cpu)
 891                         continue;
 892                 if (!cpu_online(j))
 893                         continue;
 894
 895                 dprintk("CPU %u already managed, adding link\n", j);
 896                 cpufreq_cpu_get(cpu);
 897                 cpu_sys_dev = get_cpu_sysdev(j);
 898                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 899                                         "cpufreq");
 900                 if (ret)
 901                         goto err_out_unregister;
 902         }
 903
 904         policy->governor = NULL; /* to assure that the starting sequence is
 905                                   * run in cpufreq_set_policy */
 906
 907         /* set default policy */
 908         ret = __cpufreq_set_policy(policy, &new_policy);
 909         policy->user_policy.policy = policy->policy;
 910         policy->user_policy.governor = policy->governor;
 911
 912         unlock_policy_rwsem_write(cpu);
 913
 914         if (ret) {
 915                 dprintk("setting policy failed\n");
 916                 goto err_out_unregister;
 917         }
 918
 919         kobject_uevent(&policy->kobj, KOBJ_ADD);
 920         module_put(cpufreq_driver->owner);
 921         dprintk("initialization complete\n");
 922         cpufreq_debug_enable_ratelimit();
 923
 924         return 0;
 925
 926
 927 err_out_unregister:
 928         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 929         for_each_cpu_mask(j, policy->cpus)
 930                 cpufreq_cpu_data[j] = NULL;
 931         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 932
 933         kobject_put(&policy->kobj);
 934         wait_for_completion(&policy->kobj_unregister);
 935
 936 err_out_driver_exit:
 937         if (cpufreq_driver->exit)
 938                 cpufreq_driver->exit(policy);
 939
 940 err_out:
 941         unlock_policy_rwsem_write(cpu);
 942         kfree(policy);
 943
 944 nomem_out:
 945         module_put(cpufreq_driver->owner);
 946 module_out:
 947         cpufreq_debug_enable_ratelimit();
 948         return ret;
 949 }
 950
 951
 952 /**
 953  * __cpufreq_remove_dev - remove a CPU device
 954  *
 955  * Removes the cpufreq interface for a CPU device.
 956  * Caller should already have policy_rwsem in write mode for this CPU.
 957  * This routine frees the rwsem before returning.
 958  */
 959 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
 960 {
 961         unsigned int cpu = sys_dev->id;
 962         unsigned long flags;
 963         struct cpufreq_policy *data;
 964 #ifdef CONFIG_SMP
 965         struct sys_device *cpu_sys_dev;
 966         unsigned int j;
 967 #endif
 968
 969         cpufreq_debug_disable_ratelimit();
 970         dprintk("unregistering CPU %u\n", cpu);
 971
 972         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 973         data = cpufreq_cpu_data[cpu];
 974
 975         if (!data) {
 976                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 977                 cpufreq_debug_enable_ratelimit();
 978                 unlock_policy_rwsem_write(cpu);
 979                 return -EINVAL;
 980         }
 981         cpufreq_cpu_data[cpu] = NULL;
 982
 983
 984 #ifdef CONFIG_SMP
 985         /* if this isn't the CPU which is the parent of the kobj, we
 986          * only need to unlink, put and exit
 987          */
 988         if (unlikely(cpu != data->cpu)) {
 989                 dprintk("removing link\n");
 990                 cpu_clear(cpu, data->cpus);
 991                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 992                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
 993                 cpufreq_cpu_put(data);
 994                 cpufreq_debug_enable_ratelimit();
 995                 unlock_policy_rwsem_write(cpu);
 996                 return 0;
 997         }
 998 #endif
 999
1000 #ifdef CONFIG_SMP
1001
1002 #ifdef CONFIG_HOTPLUG_CPU
1003         cpufreq_cpu_governor[cpu] = data->governor;
1004 #endif
1005
1006         /* if we have other CPUs still registered, we need to unlink them,
1007          * or else wait_for_completion below will lock up. Clean the
1008          * cpufreq_cpu_data[] while holding the lock, and remove the sysfs
1009          * links afterwards.
1010          */
1011         if (unlikely(cpus_weight(data->cpus) > 1)) {
1012                 for_each_cpu_mask(j, data->cpus) {
1013                         if (j == cpu)
1014                                 continue;
1015                         cpufreq_cpu_data[j] = NULL;
1016                 }
1017         }
1018
1019         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1020
1021         if (unlikely(cpus_weight(data->cpus) > 1)) {
1022                 for_each_cpu_mask(j, data->cpus) {
1023                         if (j == cpu)
1024                                 continue;
1025                         dprintk("removing link for cpu %u\n", j);
1026 #ifdef CONFIG_HOTPLUG_CPU
1027                         cpufreq_cpu_governor[j] = data->governor;
1028 #endif
1029                         cpu_sys_dev = get_cpu_sysdev(j);
1030                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1031                         cpufreq_cpu_put(data);
1032                 }
1033         }
1034 #else
1035         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1036 #endif
1037
1038         if (cpufreq_driver->target)
1039                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1040
1041         unlock_policy_rwsem_write(cpu);
1042
1043         kobject_put(&data->kobj);
1044
1045         /* we need to make sure that the underlying kobj is actually
1046          * not referenced anymore by anybody before we proceed with
1047          * unloading.
1048          */
1049         dprintk("waiting for dropping of refcount\n");
1050         wait_for_completion(&data->kobj_unregister);
1051         dprintk("wait complete\n");
1052
1053         if (cpufreq_driver->exit)
1054                 cpufreq_driver->exit(data);
1055
1056         kfree(data);
1057
1058         cpufreq_debug_enable_ratelimit();
1059         return 0;
1060 }
1061
1062
1063 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1064 {
1065         unsigned int cpu = sys_dev->id;
1066         int retval;
1067
1068         if (cpu_is_offline(cpu))
1069                 return 0;
1070
1071         if (unlikely(lock_policy_rwsem_write(cpu)))
1072                 BUG();
1073
1074         retval = __cpufreq_remove_dev(sys_dev);
1075         return retval;
1076 }
1077
1078
1079 static void handle_update(struct work_struct *work)
1080 {
1081         struct cpufreq_policy *policy =
1082                 container_of(work, struct cpufreq_policy, update);
1083         unsigned int cpu = policy->cpu;
1084         dprintk("handle_update for cpu %u called\n", cpu);
1085         cpufreq_update_policy(cpu);
1086 }
1087
1088 /**
1089  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1090  *      @cpu: cpu number
1091  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1092  *      @new_freq: CPU frequency the CPU actually runs at
1093  *
1094  *      We adjust to current frequency first, and need to clean up later. So either call
1095  *      to cpufreq_update_policy() or schedule handle_update()).
1096  */
1097 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1098                                 unsigned int new_freq)
1099 {
1100         struct cpufreq_freqs freqs;
1101
1102         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1103                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1104
1105         freqs.cpu = cpu;
1106         freqs.old = old_freq;
1107         freqs.new = new_freq;
1108         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1109         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1110 }
1111
1112
1113 /**
1114  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1115  * @cpu: CPU number
1116  *
1117  * This is the last known freq, without actually getting it from the driver.
1118  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1119  */
1120 unsigned int cpufreq_quick_get(unsigned int cpu)
1121 {
1122         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1123         unsigned int ret_freq = 0;
1124
1125         if (policy) {
1126                 ret_freq = policy->cur;
1127                 cpufreq_cpu_put(policy);
1128         }
1129
1130         return ret_freq;
1131 }
1132 EXPORT_SYMBOL(cpufreq_quick_get);
1133
1134
1135 static unsigned int __cpufreq_get(unsigned int cpu)
1136 {
1137         struct cpufreq_policy *policy = cpufreq_cpu_data[cpu];
1138         unsigned int ret_freq = 0;
1139
1140         if (!cpufreq_driver->get)
1141                 return ret_freq;
1142
1143         ret_freq = cpufreq_driver->get(cpu);
1144
1145         if (ret_freq && policy->cur &&
1146                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1147                 /* verify no discrepancy between actual and
1148                                         saved value exists */
1149                 if (unlikely(ret_freq != policy->cur)) {
1150                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1151                         schedule_work(&policy->update);
1152                 }
1153         }
1154
1155         return ret_freq;
1156 }
1157
1158 /**
1159  * cpufreq_get - get the current CPU frequency (in kHz)
1160  * @cpu: CPU number
1161  *
1162  * Get the CPU current (static) CPU frequency
1163  */
1164 unsigned int cpufreq_get(unsigned int cpu)
1165 {
1166         unsigned int ret_freq = 0;
1167         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1168
1169         if (!policy)
1170                 goto out;
1171
1172         if (unlikely(lock_policy_rwsem_read(cpu)))
1173                 goto out_policy;
1174
1175         ret_freq = __cpufreq_get(cpu);
1176
1177         unlock_policy_rwsem_read(cpu);
1178
1179 out_policy:
1180         cpufreq_cpu_put(policy);
1181 out:
1182         return ret_freq;
1183 }
1184 EXPORT_SYMBOL(cpufreq_get);
1185
1186
1187 /**
1188  *      cpufreq_suspend - let the low level driver prepare for suspend
1189  */
1190
1191 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1192 {
1193         int cpu = sysdev->id;
1194         int ret = 0;
1195         unsigned int cur_freq = 0;
1196         struct cpufreq_policy *cpu_policy;
1197
1198         dprintk("suspending cpu %u\n", cpu);
1199
1200         if (!cpu_online(cpu))
1201                 return 0;
1202
1203         /* we may be lax here as interrupts are off. Nonetheless
1204          * we need to grab the correct cpu policy, as to check
1205          * whether we really run on this CPU.
1206          */
1207
1208         cpu_policy = cpufreq_cpu_get(cpu);
1209         if (!cpu_policy)
1210                 return -EINVAL;
1211
1212         /* only handle each CPU group once */
1213         if (unlikely(cpu_policy->cpu != cpu))
1214                 goto out;
1215
1216         if (cpufreq_driver->suspend) {
1217                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1218                 if (ret) {
1219                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1220                                         "step on CPU %u\n", cpu_policy->cpu);
1221                         goto out;
1222                 }
1223         }
1224
1225         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1226                 goto out;
1227
1228         if (cpufreq_driver->get)
1229                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1230
1231         if (!cur_freq || !cpu_policy->cur) {
1232                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1233                        "frequency is what timing core thinks it is.\n");
1234                 goto out;
1235         }
1236
1237         if (unlikely(cur_freq != cpu_policy->cur)) {
1238                 struct cpufreq_freqs freqs;
1239
1240                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1241                         dprintk("Warning: CPU frequency is %u, "
1242                                "cpufreq assumed %u kHz.\n",
1243                                cur_freq, cpu_policy->cur);
1244
1245                 freqs.cpu = cpu;
1246                 freqs.old = cpu_policy->cur;
1247                 freqs.new = cur_freq;
1248
1249                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1250                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1251                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1252
1253                 cpu_policy->cur = cur_freq;
1254         }
1255
1256 out:
1257         cpufreq_cpu_put(cpu_policy);
1258         return ret;
1259 }
1260
1261 /**
1262  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1263  *
1264  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1265  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1266  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1267  *          restored.
1268  */
1269 static int cpufreq_resume(struct sys_device *sysdev)
1270 {
1271         int cpu = sysdev->id;
1272         int ret = 0;
1273         struct cpufreq_policy *cpu_policy;
1274
1275         dprintk("resuming cpu %u\n", cpu);
1276
1277         if (!cpu_online(cpu))
1278                 return 0;
1279
1280         /* we may be lax here as interrupts are off. Nonetheless
1281          * we need to grab the correct cpu policy, as to check
1282          * whether we really run on this CPU.
1283          */
1284
1285         cpu_policy = cpufreq_cpu_get(cpu);
1286         if (!cpu_policy)
1287                 return -EINVAL;
1288
1289         /* only handle each CPU group once */
1290         if (unlikely(cpu_policy->cpu != cpu))
1291                 goto fail;
1292
1293         if (cpufreq_driver->resume) {
1294                 ret = cpufreq_driver->resume(cpu_policy);
1295                 if (ret) {
1296                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1297                                         "step on CPU %u\n", cpu_policy->cpu);
1298                         goto fail;
1299                 }
1300         }
1301
1302         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1303                 unsigned int cur_freq = 0;
1304
1305                 if (cpufreq_driver->get)
1306                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1307
1308                 if (!cur_freq || !cpu_policy->cur) {
1309                         printk(KERN_ERR "cpufreq: resume failed to assert "
1310                                         "current frequency is what timing core "
1311                                         "thinks it is.\n");
1312                         goto out;
1313                 }
1314
1315                 if (unlikely(cur_freq != cpu_policy->cur)) {
1316                         struct cpufreq_freqs freqs;
1317
1318                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1319                                 dprintk("Warning: CPU frequency "
1320                                        "is %u, cpufreq assumed %u kHz.\n",
1321                                        cur_freq, cpu_policy->cur);
1322
1323                         freqs.cpu = cpu;
1324                         freqs.old = cpu_policy->cur;
1325                         freqs.new = cur_freq;
1326
1327                         srcu_notifier_call_chain(
1328                                         &cpufreq_transition_notifier_list,
1329                                         CPUFREQ_RESUMECHANGE, &freqs);
1330                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1331
1332                         cpu_policy->cur = cur_freq;
1333                 }
1334         }
1335
1336 out:
1337         schedule_work(&cpu_policy->update);
1338 fail:
1339         cpufreq_cpu_put(cpu_policy);
1340         return ret;
1341 }
1342
1343 static struct sysdev_driver cpufreq_sysdev_driver = {
1344         .add            = cpufreq_add_dev,
1345         .remove         = cpufreq_remove_dev,
1346         .suspend        = cpufreq_suspend,
1347         .resume         = cpufreq_resume,
1348 };
1349
1350
1351 /*********************************************************************
1352  *                     NOTIFIER LISTS INTERFACE                      *
1353  *********************************************************************/
1354
1355 /**
1356  *      cpufreq_register_notifier - register a driver with cpufreq
1357  *      @nb: notifier function to register
1358  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1359  *
1360  *      Add a driver to one of two lists: either a list of drivers that
1361  *      are notified about clock rate changes (once before and once after
1362  *      the transition), or a list of drivers that are notified about
1363  *      changes in cpufreq policy.
1364  *
1365  *      This function may sleep, and has the same return conditions as
1366  *      blocking_notifier_chain_register.
1367  */
1368 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1369 {
1370         int ret;
1371
1372         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1373
1374         switch (list) {
1375         case CPUFREQ_TRANSITION_NOTIFIER:
1376                 ret = srcu_notifier_chain_register(
1377                                 &cpufreq_transition_notifier_list, nb);
1378                 break;
1379         case CPUFREQ_POLICY_NOTIFIER:
1380                 ret = blocking_notifier_chain_register(
1381                                 &cpufreq_policy_notifier_list, nb);
1382                 break;
1383         default:
1384                 ret = -EINVAL;
1385         }
1386
1387         return ret;
1388 }
1389 EXPORT_SYMBOL(cpufreq_register_notifier);
1390
1391
1392 /**
1393  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1394  *      @nb: notifier block to be unregistered
1395  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1396  *
1397  *      Remove a driver from the CPU frequency notifier list.
1398  *
1399  *      This function may sleep, and has the same return conditions as
1400  *      blocking_notifier_chain_unregister.
1401  */
1402 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1403 {
1404         int ret;
1405
1406         switch (list) {
1407         case CPUFREQ_TRANSITION_NOTIFIER:
1408                 ret = srcu_notifier_chain_unregister(
1409                                 &cpufreq_transition_notifier_list, nb);
1410                 break;
1411         case CPUFREQ_POLICY_NOTIFIER:
1412                 ret = blocking_notifier_chain_unregister(
1413                                 &cpufreq_policy_notifier_list, nb);
1414                 break;
1415         default:
1416                 ret = -EINVAL;
1417         }
1418
1419         return ret;
1420 }
1421 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1422
1423
1424 /*********************************************************************
1425  *                              GOVERNORS                            *
1426  *********************************************************************/
1427
1428
1429 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1430                             unsigned int target_freq,
1431                             unsigned int relation)
1432 {
1433         int retval = -EINVAL;
1434
1435         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1436                 target_freq, relation);
1437         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1438                 retval = cpufreq_driver->target(policy, target_freq, relation);
1439
1440         return retval;
1441 }
1442 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1443
1444 int cpufreq_driver_target(struct cpufreq_policy *policy,
1445                           unsigned int target_freq,
1446                           unsigned int relation)
1447 {
1448         int ret;
1449
1450         policy = cpufreq_cpu_get(policy->cpu);
1451         if (!policy)
1452                 return -EINVAL;
1453
1454         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1455                 return -EINVAL;
1456
1457         ret = __cpufreq_driver_target(policy, target_freq, relation);
1458
1459         unlock_policy_rwsem_write(policy->cpu);
1460
1461         cpufreq_cpu_put(policy);
1462         return ret;
1463 }
1464 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1465
1466 int __cpufreq_driver_getavg(struct cpufreq_policy *policy)
1467 {
1468         int ret = 0;
1469
1470         policy = cpufreq_cpu_get(policy->cpu);
1471         if (!policy)
1472                 return -EINVAL;
1473
1474         if (cpu_online(policy->cpu) && cpufreq_driver->getavg)
1475                 ret = cpufreq_driver->getavg(policy->cpu);
1476
1477         cpufreq_cpu_put(policy);
1478         return ret;
1479 }
1480 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1481
1482 /*
1483  * when "event" is CPUFREQ_GOV_LIMITS
1484  */
1485
1486 static int __cpufreq_governor(struct cpufreq_policy *policy,
1487                                         unsigned int event)
1488 {
1489         int ret;
1490
1491         /* Only must be defined when default governor is known to have latency
1492            restrictions, like e.g. conservative or ondemand.
1493            That this is the case is already ensured in Kconfig
1494         */
1495 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1496         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1497 #else
1498         struct cpufreq_governor *gov = NULL;
1499 #endif
1500
1501         if (policy->governor->max_transition_latency &&
1502             policy->cpuinfo.transition_latency >
1503             policy->governor->max_transition_latency) {
1504                 if (!gov)
1505                         return -EINVAL;
1506                 else {
1507                         printk(KERN_WARNING "%s governor failed, too long"
1508                                " transition latency of HW, fallback"
1509                                " to %s governor\n",
1510                                policy->governor->name,
1511                                gov->name);
1512                         policy->governor = gov;
1513                 }
1514         }
1515
1516         if (!try_module_get(policy->governor->owner))
1517                 return -EINVAL;
1518
1519         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1520                                                 policy->cpu, event);
1521         ret = policy->governor->governor(policy, event);
1522
1523         /* we keep one module reference alive for
1524                         each CPU governed by this CPU */
1525         if ((event != CPUFREQ_GOV_START) || ret)
1526                 module_put(policy->governor->owner);
1527         if ((event == CPUFREQ_GOV_STOP) && !ret)
1528                 module_put(policy->governor->owner);
1529
1530         return ret;
1531 }
1532
1533
1534 int cpufreq_register_governor(struct cpufreq_governor *governor)
1535 {
1536         int err;
1537
1538         if (!governor)
1539                 return -EINVAL;
1540
1541         mutex_lock(&cpufreq_governor_mutex);
1542
1543         err = -EBUSY;
1544         if (__find_governor(governor->name) == NULL) {
1545                 err = 0;
1546                 list_add(&governor->governor_list, &cpufreq_governor_list);
1547         }
1548
1549         mutex_unlock(&cpufreq_governor_mutex);
1550         return err;
1551 }
1552 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1553
1554
1555 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1556 {
1557         if (!governor)
1558                 return;
1559
1560         mutex_lock(&cpufreq_governor_mutex);
1561         list_del(&governor->governor_list);
1562         mutex_unlock(&cpufreq_governor_mutex);
1563         return;
1564 }
1565 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1566
1567
1568
1569 /*********************************************************************
1570  *                          POLICY INTERFACE                         *
1571  *********************************************************************/
1572
1573 /**
1574  * cpufreq_get_policy - get the current cpufreq_policy
1575  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1576  *
1577  * Reads the current cpufreq policy.
1578  */
1579 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1580 {
1581         struct cpufreq_policy *cpu_policy;
1582         if (!policy)
1583                 return -EINVAL;
1584
1585         cpu_policy = cpufreq_cpu_get(cpu);
1586         if (!cpu_policy)
1587                 return -EINVAL;
1588
1589         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1590
1591         cpufreq_cpu_put(cpu_policy);
1592         return 0;
1593 }
1594 EXPORT_SYMBOL(cpufreq_get_policy);
1595
1596
1597 /*
1598  * data   : current policy.
1599  * policy : policy to be set.
1600  */
1601 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1602                                 struct cpufreq_policy *policy)
1603 {
1604         int ret = 0;
1605
1606         cpufreq_debug_disable_ratelimit();
1607         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1608                 policy->min, policy->max);
1609
1610         memcpy(&policy->cpuinfo, &data->cpuinfo,
1611                                 sizeof(struct cpufreq_cpuinfo));
1612
1613         if (policy->min > data->max || policy->max < data->min) {
1614                 ret = -EINVAL;
1615                 goto error_out;
1616         }
1617
1618         /* verify the cpu speed can be set within this limit */
1619         ret = cpufreq_driver->verify(policy);
1620         if (ret)
1621                 goto error_out;
1622
1623         /* adjust if necessary - all reasons */
1624         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1625                         CPUFREQ_ADJUST, policy);
1626
1627         /* adjust if necessary - hardware incompatibility*/
1628         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1629                         CPUFREQ_INCOMPATIBLE, policy);
1630
1631         /* verify the cpu speed can be set within this limit,
1632            which might be different to the first one */
1633         ret = cpufreq_driver->verify(policy);
1634         if (ret)
1635                 goto error_out;
1636
1637         /* notification of the new policy */
1638         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1639                         CPUFREQ_NOTIFY, policy);
1640
1641         data->min = policy->min;
1642         data->max = policy->max;
1643
1644         dprintk("new min and max freqs are %u - %u kHz\n",
1645                                         data->min, data->max);
1646
1647         if (cpufreq_driver->setpolicy) {
1648                 data->policy = policy->policy;
1649                 dprintk("setting range\n");
1650                 ret = cpufreq_driver->setpolicy(policy);
1651         } else {
1652                 if (policy->governor != data->governor) {
1653                         /* save old, working values */
1654                         struct cpufreq_governor *old_gov = data->governor;
1655
1656                         dprintk("governor switch\n");
1657
1658                         /* end old governor */
1659                         if (data->governor)
1660                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1661
1662                         /* start new governor */
1663                         data->governor = policy->governor;
1664                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1665                                 /* new governor failed, so re-start old one */
1666                                 dprintk("starting governor %s failed\n",
1667                                                         data->governor->name);
1668                                 if (old_gov) {
1669                                         data->governor = old_gov;
1670                                         __cpufreq_governor(data,
1671                                                            CPUFREQ_GOV_START);
1672                                 }
1673                                 ret = -EINVAL;
1674                                 goto error_out;
1675                         }
1676                         /* might be a policy change, too, so fall through */
1677                 }
1678                 dprintk("governor: change or update limits\n");
1679                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1680         }
1681
1682 error_out:
1683         cpufreq_debug_enable_ratelimit();
1684         return ret;
1685 }
1686
1687 /**
1688  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1689  *      @cpu: CPU which shall be re-evaluated
1690  *
1691  *      Usefull for policy notifiers which have different necessities
1692  *      at different times.
1693  */
1694 int cpufreq_update_policy(unsigned int cpu)
1695 {
1696         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1697         struct cpufreq_policy policy;
1698         int ret = 0;
1699
1700         if (!data)
1701                 return -ENODEV;
1702
1703         if (unlikely(lock_policy_rwsem_write(cpu)))
1704                 return -EINVAL;
1705
1706         dprintk("updating policy for CPU %u\n", cpu);
1707         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1708         policy.min = data->user_policy.min;
1709         policy.max = data->user_policy.max;
1710         policy.policy = data->user_policy.policy;
1711         policy.governor = data->user_policy.governor;
1712
1713         /* BIOS might change freq behind our back
1714           -> ask driver for current freq and notify governors about a change */
1715         if (cpufreq_driver->get) {
1716                 policy.cur = cpufreq_driver->get(cpu);
1717                 if (!data->cur) {
1718                         dprintk("Driver did not initialize current freq");
1719                         data->cur = policy.cur;
1720                 } else {
1721                         if (data->cur != policy.cur)
1722                                 cpufreq_out_of_sync(cpu, data->cur,
1723                                                                 policy.cur);
1724                 }
1725         }
1726
1727         ret = __cpufreq_set_policy(data, &policy);
1728
1729         unlock_policy_rwsem_write(cpu);
1730
1731         cpufreq_cpu_put(data);
1732         return ret;
1733 }
1734 EXPORT_SYMBOL(cpufreq_update_policy);
1735
1736 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1737                                         unsigned long action, void *hcpu)
1738 {
1739         unsigned int cpu = (unsigned long)hcpu;
1740         struct sys_device *sys_dev;
1741
1742         sys_dev = get_cpu_sysdev(cpu);
1743         if (sys_dev) {
1744                 switch (action) {
1745                 case CPU_ONLINE:
1746                 case CPU_ONLINE_FROZEN:
1747                         cpufreq_add_dev(sys_dev);
1748                         break;
1749                 case CPU_DOWN_PREPARE:
1750                 case CPU_DOWN_PREPARE_FROZEN:
1751                         if (unlikely(lock_policy_rwsem_write(cpu)))
1752                                 BUG();
1753
1754                         __cpufreq_remove_dev(sys_dev);
1755                         break;
1756                 case CPU_DOWN_FAILED:
1757                 case CPU_DOWN_FAILED_FROZEN:
1758                         cpufreq_add_dev(sys_dev);
1759                         break;
1760                 }
1761         }
1762         return NOTIFY_OK;
1763 }
1764
1765 static struct notifier_block __refdata cpufreq_cpu_notifier =
1766 {
1767     .notifier_call = cpufreq_cpu_callback,
1768 };
1769
1770 /*********************************************************************
1771  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1772  *********************************************************************/
1773
1774 /**
1775  * cpufreq_register_driver - register a CPU Frequency driver
1776  * @driver_data: A struct cpufreq_driver containing the values#
1777  * submitted by the CPU Frequency driver.
1778  *
1779  *   Registers a CPU Frequency driver to this core code. This code
1780  * returns zero on success, -EBUSY when another driver got here first
1781  * (and isn't unregistered in the meantime).
1782  *
1783  */
1784 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1785 {
1786         unsigned long flags;
1787         int ret;
1788
1789         if (!driver_data || !driver_data->verify || !driver_data->init ||
1790             ((!driver_data->setpolicy) && (!driver_data->target)))
1791                 return -EINVAL;
1792
1793         dprintk("trying to register driver %s\n", driver_data->name);
1794
1795         if (driver_data->setpolicy)
1796                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1797
1798         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1799         if (cpufreq_driver) {
1800                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1801                 return -EBUSY;
1802         }
1803         cpufreq_driver = driver_data;
1804         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1805
1806         ret = sysdev_driver_register(&cpu_sysdev_class,&cpufreq_sysdev_driver);
1807
1808         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1809                 int i;
1810                 ret = -ENODEV;
1811
1812                 /* check for at least one working CPU */
1813                 for (i=0; i<NR_CPUS; i++)
1814                         if (cpufreq_cpu_data[i])
1815                                 ret = 0;
1816
1817                 /* if all ->init() calls failed, unregister */
1818                 if (ret) {
1819                         dprintk("no CPU initialized for driver %s\n",
1820                                                         driver_data->name);
1821                         sysdev_driver_unregister(&cpu_sysdev_class,
1822                                                 &cpufreq_sysdev_driver);
1823
1824                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1825                         cpufreq_driver = NULL;
1826                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1827                 }
1828         }
1829
1830         if (!ret) {
1831                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1832                 dprintk("driver %s up and running\n", driver_data->name);
1833                 cpufreq_debug_enable_ratelimit();
1834         }
1835
1836         return ret;
1837 }
1838 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1839
1840
1841 /**
1842  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1843  *
1844  *    Unregister the current CPUFreq driver. Only call this if you have
1845  * the right to do so, i.e. if you have succeeded in initialising before!
1846  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1847  * currently not initialised.
1848  */
1849 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1850 {
1851         unsigned long flags;
1852
1853         cpufreq_debug_disable_ratelimit();
1854
1855         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1856                 cpufreq_debug_enable_ratelimit();
1857                 return -EINVAL;
1858         }
1859
1860         dprintk("unregistering driver %s\n", driver->name);
1861
1862         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1863         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1864
1865         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1866         cpufreq_driver = NULL;
1867         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1868
1869         return 0;
1870 }
1871 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1872
1873 static int __init cpufreq_core_init(void)
1874 {
1875         int cpu;
1876
1877         for_each_possible_cpu(cpu) {
1878                 per_cpu(policy_cpu, cpu) = -1;
1879                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1880         }
1881         return 0;
1882 }
1883
1884 core_initcall(cpufreq_core_init);