SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static struct cpufreq_policy *cpufreq_cpu_data[NR_CPUS];
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static struct cpufreq_governor *cpufreq_cpu_governor[NR_CPUS];
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static int __init init_cpufreq_transition_notifier_list(void)
 122 {
 123         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 124         return 0;
 125 }
 126 pure_initcall(init_cpufreq_transition_notifier_list);
 127
 128 static LIST_HEAD(cpufreq_governor_list);
 129 static DEFINE_MUTEX (cpufreq_governor_mutex);
 130
 131 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 132 {
 133         struct cpufreq_policy *data;
 134         unsigned long flags;
 135
 136         if (cpu >= NR_CPUS)
 137                 goto err_out;
 138
 139         /* get the cpufreq driver */
 140         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 141
 142         if (!cpufreq_driver)
 143                 goto err_out_unlock;
 144
 145         if (!try_module_get(cpufreq_driver->owner))
 146                 goto err_out_unlock;
 147
 148
 149         /* get the CPU */
 150         data = cpufreq_cpu_data[cpu];
 151
 152         if (!data)
 153                 goto err_out_put_module;
 154
 155         if (!kobject_get(&data->kobj))
 156                 goto err_out_put_module;
 157
 158         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 159         return data;
 160
 161 err_out_put_module:
 162         module_put(cpufreq_driver->owner);
 163 err_out_unlock:
 164         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 165 err_out:
 166         return NULL;
 167 }
 168 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 169
 170
 171 void cpufreq_cpu_put(struct cpufreq_policy *data)
 172 {
 173         kobject_put(&data->kobj);
 174         module_put(cpufreq_driver->owner);
 175 }
 176 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 177
 178
 179 /*********************************************************************
 180  *                     UNIFIED DEBUG HELPERS                         *
 181  *********************************************************************/
 182 #ifdef CONFIG_CPU_FREQ_DEBUG
 183
 184 /* what part(s) of the CPUfreq subsystem are debugged? */
 185 static unsigned int debug;
 186
 187 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 188  * set or modify this value.
 189  */
 190 static unsigned int debug_ratelimit = 1;
 191
 192 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 193  * loading of a cpufreq driver, temporarily disabled when a new policy
 194  * is set, and disabled upon cpufreq driver removal
 195  */
 196 static unsigned int disable_ratelimit = 1;
 197 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 198
 199 static void cpufreq_debug_enable_ratelimit(void)
 200 {
 201         unsigned long flags;
 202
 203         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 204         if (disable_ratelimit)
 205                 disable_ratelimit--;
 206         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 207 }
 208
 209 static void cpufreq_debug_disable_ratelimit(void)
 210 {
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 214         disable_ratelimit++;
 215         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 216 }
 217
 218 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 219                                                         const char *fmt, ...)
 220 {
 221         char s[256];
 222         va_list args;
 223         unsigned int len;
 224         unsigned long flags;
 225
 226         WARN_ON(!prefix);
 227         if (type & debug) {
 228                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 229                 if (!disable_ratelimit && debug_ratelimit
 230                                         && !printk_ratelimit()) {
 231                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 232                         return;
 233                 }
 234                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235
 236                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 237
 238                 va_start(args, fmt);
 239                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 240                 va_end(args);
 241
 242                 printk(s);
 243
 244                 WARN_ON(len < 5);
 245         }
 246 }
 247 EXPORT_SYMBOL(cpufreq_debug_printk);
 248
 249
 250 module_param(debug, uint, 0644);
 251 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 252                         " 2 to debug drivers, and 4 to debug governors.");
 253
 254 module_param(debug_ratelimit, uint, 0644);
 255 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 256                                         " set to 0 to disable ratelimiting.");
 257
 258 #else /* !CONFIG_CPU_FREQ_DEBUG */
 259
 260 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 261 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 262
 263 #endif /* CONFIG_CPU_FREQ_DEBUG */
 264
 265
 266 /*********************************************************************
 267  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 268  *********************************************************************/
 269
 270 /**
 271  * adjust_jiffies - adjust the system "loops_per_jiffy"
 272  *
 273  * This function alters the system "loops_per_jiffy" for the clock
 274  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 275  * systems as each CPU might be scaled differently. So, use the arch
 276  * per-CPU loops_per_jiffy value wherever possible.
 277  */
 278 #ifndef CONFIG_SMP
 279 static unsigned long l_p_j_ref;
 280 static unsigned int  l_p_j_ref_freq;
 281
 282 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 283 {
 284         if (ci->flags & CPUFREQ_CONST_LOOPS)
 285                 return;
 286
 287         if (!l_p_j_ref_freq) {
 288                 l_p_j_ref = loops_per_jiffy;
 289                 l_p_j_ref_freq = ci->old;
 290                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 291                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 292         }
 293         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 294             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 295             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 296                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 297                                                                 ci->new);
 298                 dprintk("scaling loops_per_jiffy to %lu "
 299                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 300         }
 301 }
 302 #else
 303 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 304 {
 305         return;
 306 }
 307 #endif
 308
 309
 310 /**
 311  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 312  * on frequency transition.
 313  *
 314  * This function calls the transition notifiers and the "adjust_jiffies"
 315  * function. It is called twice on all CPU frequency changes that have
 316  * external effects.
 317  */
 318 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 319 {
 320         struct cpufreq_policy *policy;
 321
 322         BUG_ON(irqs_disabled());
 323
 324         freqs->flags = cpufreq_driver->flags;
 325         dprintk("notification %u of frequency transition to %u kHz\n",
 326                 state, freqs->new);
 327
 328         policy = cpufreq_cpu_data[freqs->cpu];
 329         switch (state) {
 330
 331         case CPUFREQ_PRECHANGE:
 332                 /* detect if the driver reported a value as "old frequency"
 333                  * which is not equal to what the cpufreq core thinks is
 334                  * "old frequency".
 335                  */
 336                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 337                         if ((policy) && (policy->cpu == freqs->cpu) &&
 338                             (policy->cur) && (policy->cur != freqs->old)) {
 339                                 dprintk("Warning: CPU frequency is"
 340                                         " %u, cpufreq assumed %u kHz.\n",
 341                                         freqs->old, policy->cur);
 342                                 freqs->old = policy->cur;
 343                         }
 344                 }
 345                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 346                                 CPUFREQ_PRECHANGE, freqs);
 347                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 348                 break;
 349
 350         case CPUFREQ_POSTCHANGE:
 351                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 352                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 353                                 CPUFREQ_POSTCHANGE, freqs);
 354                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 355                         policy->cur = freqs->new;
 356                 break;
 357         }
 358 }
 359 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 360
 361
 362
 363 /*********************************************************************
 364  *                          SYSFS INTERFACE                          *
 365  *********************************************************************/
 366
 367 static struct cpufreq_governor *__find_governor(const char *str_governor)
 368 {
 369         struct cpufreq_governor *t;
 370
 371         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 372                 if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 373                         return t;
 374
 375         return NULL;
 376 }
 377
 378 /**
 379  * cpufreq_parse_governor - parse a governor string
 380  */
 381 static int cpufreq_parse_governor (char *str_governor, unsigned int *policy,
 382                                 struct cpufreq_governor **governor)
 383 {
 384         int err = -EINVAL;
 385
 386         if (!cpufreq_driver)
 387                 goto out;
 388
 389         if (cpufreq_driver->setpolicy) {
 390                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 391                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 392                         err = 0;
 393                 } else if (!strnicmp(str_governor, "powersave",
 394                                                 CPUFREQ_NAME_LEN)) {
 395                         *policy = CPUFREQ_POLICY_POWERSAVE;
 396                         err = 0;
 397                 }
 398         } else if (cpufreq_driver->target) {
 399                 struct cpufreq_governor *t;
 400
 401                 mutex_lock(&cpufreq_governor_mutex);
 402
 403                 t = __find_governor(str_governor);
 404
 405                 if (t == NULL) {
 406                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 407                                                                 str_governor);
 408
 409                         if (name) {
 410                                 int ret;
 411
 412                                 mutex_unlock(&cpufreq_governor_mutex);
 413                                 ret = request_module(name);
 414                                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                                 if (ret == 0)
 417                                         t = __find_governor(str_governor);
 418                         }
 419
 420                         kfree(name);
 421                 }
 422
 423                 if (t != NULL) {
 424                         *governor = t;
 425                         err = 0;
 426                 }
 427
 428                 mutex_unlock(&cpufreq_governor_mutex);
 429         }
 430   out:
 431         return err;
 432 }
 433
 434
 435 /* drivers/base/cpu.c */
 436 extern struct sysdev_class cpu_sysdev_class;
 437
 438
 439 /**
 440  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 441  * print out cpufreq information
 442  *
 443  * Write out information from cpufreq_driver->policy[cpu]; object must be
 444  * "unsigned int".
 445  */
 446
 447 #define show_one(file_name, object)                     \
 448 static ssize_t show_##file_name                         \
 449 (struct cpufreq_policy * policy, char *buf)             \
 450 {                                                       \
 451         return sprintf (buf, "%u\n", policy->object);   \
 452 }
 453
 454 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 455 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy * policy, const char *buf, size_t count)         \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf (buf, "%u", &new_policy.object);                   \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq,min);
 488 store_one(scaling_max_freq,max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq (struct cpufreq_policy * policy,
 494                                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor (struct cpufreq_policy * policy,
 507                                                         char *buf)
 508 {
 509         if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor (struct cpufreq_policy * policy,
 523                                        const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf (buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver (struct cpufreq_policy * policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors (struct cpufreq_policy *policy,
 566                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 578                         goto out;
 579                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 580         }
 581 out:
 582         i += sprintf(&buf[i], "\n");
 583         return i;
 584 }
 585 /**
 586  * show_affected_cpus - show the CPUs affected by each transition
 587  */
 588 static ssize_t show_affected_cpus (struct cpufreq_policy * policy, char *buf)
 589 {
 590         ssize_t i = 0;
 591         unsigned int cpu;
 592
 593         for_each_cpu_mask(cpu, policy->cpus) {
 594                 if (i)
 595                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 596                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 597                 if (i >= (PAGE_SIZE - 5))
 598                     break;
 599         }
 600         i += sprintf(&buf[i], "\n");
 601         return i;
 602 }
 603
 604 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 605                 const char *buf, size_t count)
 606 {
 607         unsigned int freq = 0;
 608         unsigned int ret;
 609
 610         if (!policy->governor->store_setspeed)
 611                 return -EINVAL;
 612
 613         ret = sscanf(buf, "%u", &freq);
 614         if (ret != 1)
 615                 return -EINVAL;
 616
 617         policy->governor->store_setspeed(policy, freq);
 618
 619         return count;
 620 }
 621
 622 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 623 {
 624         if (!policy->governor->show_setspeed)
 625                 return sprintf(buf, "<unsupported>\n");
 626
 627         return policy->governor->show_setspeed(policy, buf);
 628 }
 629
 630 #define define_one_ro(_name) \
 631 static struct freq_attr _name = \
 632 __ATTR(_name, 0444, show_##_name, NULL)
 633
 634 #define define_one_ro0400(_name) \
 635 static struct freq_attr _name = \
 636 __ATTR(_name, 0400, show_##_name, NULL)
 637
 638 #define define_one_rw(_name) \
 639 static struct freq_attr _name = \
 640 __ATTR(_name, 0644, show_##_name, store_##_name)
 641
 642 define_one_ro0400(cpuinfo_cur_freq);
 643 define_one_ro(cpuinfo_min_freq);
 644 define_one_ro(cpuinfo_max_freq);
 645 define_one_ro(scaling_available_governors);
 646 define_one_ro(scaling_driver);
 647 define_one_ro(scaling_cur_freq);
 648 define_one_ro(affected_cpus);
 649 define_one_rw(scaling_min_freq);
 650 define_one_rw(scaling_max_freq);
 651 define_one_rw(scaling_governor);
 652 define_one_rw(scaling_setspeed);
 653
 654 static struct attribute * default_attrs[] = {
 655         &cpuinfo_min_freq.attr,
 656         &cpuinfo_max_freq.attr,
 657         &scaling_min_freq.attr,
 658         &scaling_max_freq.attr,
 659         &affected_cpus.attr,
 660         &scaling_governor.attr,
 661         &scaling_driver.attr,
 662         &scaling_available_governors.attr,
 663         &scaling_setspeed.attr,
 664         NULL
 665 };
 666
 667 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 668 #define to_attr(a) container_of(a,struct freq_attr,attr)
 669
 670 static ssize_t show(struct kobject * kobj, struct attribute * attr ,char * buf)
 671 {
 672         struct cpufreq_policy * policy = to_policy(kobj);
 673         struct freq_attr * fattr = to_attr(attr);
 674         ssize_t ret = -EINVAL;
 675         policy = cpufreq_cpu_get(policy->cpu);
 676         if (!policy)
 677                 goto no_policy;
 678
 679         if (lock_policy_rwsem_read(policy->cpu) < 0)
 680                 goto fail;
 681
 682         if (fattr->show)
 683                 ret = fattr->show(policy, buf);
 684         else
 685                 ret = -EIO;
 686
 687         unlock_policy_rwsem_read(policy->cpu);
 688 fail:
 689         cpufreq_cpu_put(policy);
 690 no_policy:
 691         return ret;
 692 }
 693
 694 static ssize_t store(struct kobject * kobj, struct attribute * attr,
 695                      const char * buf, size_t count)
 696 {
 697         struct cpufreq_policy * policy = to_policy(kobj);
 698         struct freq_attr * fattr = to_attr(attr);
 699         ssize_t ret;
 700         policy = cpufreq_cpu_get(policy->cpu);
 701         if (!policy)
 702                 return -EINVAL;
 703
 704         if (lock_policy_rwsem_write(policy->cpu) < 0)
 705                 return -EINVAL;
 706
 707         if (fattr->store)
 708                 ret = fattr->store(policy, buf, count);
 709         else
 710                 ret = -EIO;
 711
 712         unlock_policy_rwsem_write(policy->cpu);
 713
 714         cpufreq_cpu_put(policy);
 715         return ret;
 716 }
 717
 718 static void cpufreq_sysfs_release(struct kobject * kobj)
 719 {
 720         struct cpufreq_policy * policy = to_policy(kobj);
 721         dprintk("last reference is dropped\n");
 722         complete(&policy->kobj_unregister);
 723 }
 724
 725 static struct sysfs_ops sysfs_ops = {
 726         .show   = show,
 727         .store  = store,
 728 };
 729
 730 static struct kobj_type ktype_cpufreq = {
 731         .sysfs_ops      = &sysfs_ops,
 732         .default_attrs  = default_attrs,
 733         .release        = cpufreq_sysfs_release,
 734 };
 735
 736
 737 /**
 738  * cpufreq_add_dev - add a CPU device
 739  *
 740  * Adds the cpufreq interface for a CPU device.
 741  */
 742 static int cpufreq_add_dev (struct sys_device * sys_dev)
 743 {
 744         unsigned int cpu = sys_dev->id;
 745         int ret = 0;
 746         struct cpufreq_policy new_policy;
 747         struct cpufreq_policy *policy;
 748         struct freq_attr **drv_attr;
 749         struct sys_device *cpu_sys_dev;
 750         unsigned long flags;
 751         unsigned int j;
 752 #ifdef CONFIG_SMP
 753         struct cpufreq_policy *managed_policy;
 754 #endif
 755
 756         if (cpu_is_offline(cpu))
 757                 return 0;
 758
 759         cpufreq_debug_disable_ratelimit();
 760         dprintk("adding CPU %u\n", cpu);
 761
 762 #ifdef CONFIG_SMP
 763         /* check whether a different CPU already registered this
 764          * CPU because it is in the same boat. */
 765         policy = cpufreq_cpu_get(cpu);
 766         if (unlikely(policy)) {
 767                 cpufreq_cpu_put(policy);
 768                 cpufreq_debug_enable_ratelimit();
 769                 return 0;
 770         }
 771 #endif
 772
 773         if (!try_module_get(cpufreq_driver->owner)) {
 774                 ret = -EINVAL;
 775                 goto module_out;
 776         }
 777
 778         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 779         if (!policy) {
 780                 ret = -ENOMEM;
 781                 goto nomem_out;
 782         }
 783
 784         policy->cpu = cpu;
 785         policy->cpus = cpumask_of_cpu(cpu);
 786
 787         /* Initially set CPU itself as the policy_cpu */
 788         per_cpu(policy_cpu, cpu) = cpu;
 789         lock_policy_rwsem_write(cpu);
 790
 791         init_completion(&policy->kobj_unregister);
 792         INIT_WORK(&policy->update, handle_update);
 793
 794         /* Set governor before ->init, so that driver could check it */
 795         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 796         /* call driver. From then on the cpufreq must be able
 797          * to accept all calls to ->verify and ->setpolicy for this CPU
 798          */
 799         ret = cpufreq_driver->init(policy);
 800         if (ret) {
 801                 dprintk("initialization failed\n");
 802                 unlock_policy_rwsem_write(cpu);
 803                 goto err_out;
 804         }
 805         policy->user_policy.min = policy->cpuinfo.min_freq;
 806         policy->user_policy.max = policy->cpuinfo.max_freq;
 807
 808 #ifdef CONFIG_SMP
 809
 810 #ifdef CONFIG_HOTPLUG_CPU
 811         if (cpufreq_cpu_governor[cpu]){
 812                 policy->governor = cpufreq_cpu_governor[cpu];
 813                 dprintk("Restoring governor %s for cpu %d\n",
 814                        policy->governor->name, cpu);
 815         }
 816 #endif
 817
 818         for_each_cpu_mask(j, policy->cpus) {
 819                 if (cpu == j)
 820                         continue;
 821
 822                 /* check for existing affected CPUs.  They may not be aware
 823                  * of it due to CPU Hotplug.
 824                  */
 825                 managed_policy = cpufreq_cpu_get(j);
 826                 if (unlikely(managed_policy)) {
 827
 828                         /* Set proper policy_cpu */
 829                         unlock_policy_rwsem_write(cpu);
 830                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 831
 832                         if (lock_policy_rwsem_write(cpu) < 0)
 833                                 goto err_out_driver_exit;
 834
 835                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 836                         managed_policy->cpus = policy->cpus;
 837                         cpufreq_cpu_data[cpu] = managed_policy;
 838                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 839
 840                         dprintk("CPU already managed, adding link\n");
 841                         ret = sysfs_create_link(&sys_dev->kobj,
 842                                                 &managed_policy->kobj,
 843                                                 "cpufreq");
 844                         if (ret) {
 845                                 unlock_policy_rwsem_write(cpu);
 846                                 goto err_out_driver_exit;
 847                         }
 848
 849                         cpufreq_debug_enable_ratelimit();
 850                         ret = 0;
 851                         unlock_policy_rwsem_write(cpu);
 852                         goto err_out_driver_exit; /* call driver->exit() */
 853                 }
 854         }
 855 #endif
 856         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 857
 858         /* prepare interface data */
 859         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 860                                    "cpufreq");
 861         if (ret) {
 862                 unlock_policy_rwsem_write(cpu);
 863                 goto err_out_driver_exit;
 864         }
 865         /* set up files for this cpu device */
 866         drv_attr = cpufreq_driver->attr;
 867         while ((drv_attr) && (*drv_attr)) {
 868                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 869                 if (ret) {
 870                         unlock_policy_rwsem_write(cpu);
 871                         goto err_out_driver_exit;
 872                 }
 873                 drv_attr++;
 874         }
 875         if (cpufreq_driver->get){
 876                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 877                 if (ret) {
 878                         unlock_policy_rwsem_write(cpu);
 879                         goto err_out_driver_exit;
 880                 }
 881         }
 882         if (cpufreq_driver->target){
 883                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 884                 if (ret) {
 885                         unlock_policy_rwsem_write(cpu);
 886                         goto err_out_driver_exit;
 887                 }
 888         }
 889
 890         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 891         for_each_cpu_mask(j, policy->cpus) {
 892                 cpufreq_cpu_data[j] = policy;
 893                 per_cpu(policy_cpu, j) = policy->cpu;
 894         }
 895         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 896
 897         /* symlink affected CPUs */
 898         for_each_cpu_mask(j, policy->cpus) {
 899                 if (j == cpu)
 900                         continue;
 901                 if (!cpu_online(j))
 902                         continue;
 903
 904                 dprintk("CPU %u already managed, adding link\n", j);
 905                 cpufreq_cpu_get(cpu);
 906                 cpu_sys_dev = get_cpu_sysdev(j);
 907                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 908                                         "cpufreq");
 909                 if (ret) {
 910                         unlock_policy_rwsem_write(cpu);
 911                         goto err_out_unregister;
 912                 }
 913         }
 914
 915         policy->governor = NULL; /* to assure that the starting sequence is
 916                                   * run in cpufreq_set_policy */
 917
 918         /* set default policy */
 919         ret = __cpufreq_set_policy(policy, &new_policy);
 920         policy->user_policy.policy = policy->policy;
 921         policy->user_policy.governor = policy->governor;
 922
 923         unlock_policy_rwsem_write(cpu);
 924
 925         if (ret) {
 926                 dprintk("setting policy failed\n");
 927                 goto err_out_unregister;
 928         }
 929
 930         kobject_uevent(&policy->kobj, KOBJ_ADD);
 931         module_put(cpufreq_driver->owner);
 932         dprintk("initialization complete\n");
 933         cpufreq_debug_enable_ratelimit();
 934
 935         return 0;
 936
 937
 938 err_out_unregister:
 939         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 940         for_each_cpu_mask(j, policy->cpus)
 941                 cpufreq_cpu_data[j] = NULL;
 942         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 943
 944         kobject_put(&policy->kobj);
 945         wait_for_completion(&policy->kobj_unregister);
 946
 947 err_out_driver_exit:
 948         if (cpufreq_driver->exit)
 949                 cpufreq_driver->exit(policy);
 950
 951 err_out:
 952         kfree(policy);
 953
 954 nomem_out:
 955         module_put(cpufreq_driver->owner);
 956 module_out:
 957         cpufreq_debug_enable_ratelimit();
 958         return ret;
 959 }
 960
 961
 962 /**
 963  * __cpufreq_remove_dev - remove a CPU device
 964  *
 965  * Removes the cpufreq interface for a CPU device.
 966  * Caller should already have policy_rwsem in write mode for this CPU.
 967  * This routine frees the rwsem before returning.
 968  */
 969 static int __cpufreq_remove_dev (struct sys_device * sys_dev)
 970 {
 971         unsigned int cpu = sys_dev->id;
 972         unsigned long flags;
 973         struct cpufreq_policy *data;
 974 #ifdef CONFIG_SMP
 975         struct sys_device *cpu_sys_dev;
 976         unsigned int j;
 977 #endif
 978
 979         cpufreq_debug_disable_ratelimit();
 980         dprintk("unregistering CPU %u\n", cpu);
 981
 982         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 983         data = cpufreq_cpu_data[cpu];
 984
 985         if (!data) {
 986                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 987                 cpufreq_debug_enable_ratelimit();
 988                 unlock_policy_rwsem_write(cpu);
 989                 return -EINVAL;
 990         }
 991         cpufreq_cpu_data[cpu] = NULL;
 992
 993
 994 #ifdef CONFIG_SMP
 995         /* if this isn't the CPU which is the parent of the kobj, we
 996          * only need to unlink, put and exit
 997          */
 998         if (unlikely(cpu != data->cpu)) {
 999                 dprintk("removing link\n");
1000                 cpu_clear(cpu, data->cpus);
1001                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1002                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1003                 cpufreq_cpu_put(data);
1004                 cpufreq_debug_enable_ratelimit();
1005                 unlock_policy_rwsem_write(cpu);
1006                 return 0;
1007         }
1008 #endif
1009
1010 #ifdef CONFIG_SMP
1011
1012 #ifdef CONFIG_HOTPLUG_CPU
1013         cpufreq_cpu_governor[cpu] = data->governor;
1014 #endif
1015
1016         /* if we have other CPUs still registered, we need to unlink them,
1017          * or else wait_for_completion below will lock up. Clean the
1018          * cpufreq_cpu_data[] while holding the lock, and remove the sysfs
1019          * links afterwards.
1020          */
1021         if (unlikely(cpus_weight(data->cpus) > 1)) {
1022                 for_each_cpu_mask(j, data->cpus) {
1023                         if (j == cpu)
1024                                 continue;
1025                         cpufreq_cpu_data[j] = NULL;
1026                 }
1027         }
1028
1029         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1030
1031         if (unlikely(cpus_weight(data->cpus) > 1)) {
1032                 for_each_cpu_mask(j, data->cpus) {
1033                         if (j == cpu)
1034                                 continue;
1035                         dprintk("removing link for cpu %u\n", j);
1036 #ifdef CONFIG_HOTPLUG_CPU
1037                         cpufreq_cpu_governor[j] = data->governor;
1038 #endif
1039                         cpu_sys_dev = get_cpu_sysdev(j);
1040                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1041                         cpufreq_cpu_put(data);
1042                 }
1043         }
1044 #else
1045         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1046 #endif
1047
1048         if (cpufreq_driver->target)
1049                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1050
1051         unlock_policy_rwsem_write(cpu);
1052
1053         kobject_put(&data->kobj);
1054
1055         /* we need to make sure that the underlying kobj is actually
1056          * not referenced anymore by anybody before we proceed with
1057          * unloading.
1058          */
1059         dprintk("waiting for dropping of refcount\n");
1060         wait_for_completion(&data->kobj_unregister);
1061         dprintk("wait complete\n");
1062
1063         if (cpufreq_driver->exit)
1064                 cpufreq_driver->exit(data);
1065
1066         kfree(data);
1067
1068         cpufreq_debug_enable_ratelimit();
1069         return 0;
1070 }
1071
1072
1073 static int cpufreq_remove_dev (struct sys_device * sys_dev)
1074 {
1075         unsigned int cpu = sys_dev->id;
1076         int retval;
1077
1078         if (cpu_is_offline(cpu))
1079                 return 0;
1080
1081         if (unlikely(lock_policy_rwsem_write(cpu)))
1082                 BUG();
1083
1084         retval = __cpufreq_remove_dev(sys_dev);
1085         return retval;
1086 }
1087
1088
1089 static void handle_update(struct work_struct *work)
1090 {
1091         struct cpufreq_policy *policy =
1092                 container_of(work, struct cpufreq_policy, update);
1093         unsigned int cpu = policy->cpu;
1094         dprintk("handle_update for cpu %u called\n", cpu);
1095         cpufreq_update_policy(cpu);
1096 }
1097
1098 /**
1099  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1100  *      @cpu: cpu number
1101  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1102  *      @new_freq: CPU frequency the CPU actually runs at
1103  *
1104  *      We adjust to current frequency first, and need to clean up later. So either call
1105  *      to cpufreq_update_policy() or schedule handle_update()).
1106  */
1107 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1108                                 unsigned int new_freq)
1109 {
1110         struct cpufreq_freqs freqs;
1111
1112         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1113                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1114
1115         freqs.cpu = cpu;
1116         freqs.old = old_freq;
1117         freqs.new = new_freq;
1118         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1119         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1120 }
1121
1122
1123 /**
1124  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1125  * @cpu: CPU number
1126  *
1127  * This is the last known freq, without actually getting it from the driver.
1128  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1129  */
1130 unsigned int cpufreq_quick_get(unsigned int cpu)
1131 {
1132         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1133         unsigned int ret_freq = 0;
1134
1135         if (policy) {
1136                 ret_freq = policy->cur;
1137                 cpufreq_cpu_put(policy);
1138         }
1139
1140         return (ret_freq);
1141 }
1142 EXPORT_SYMBOL(cpufreq_quick_get);
1143
1144
1145 static unsigned int __cpufreq_get(unsigned int cpu)
1146 {
1147         struct cpufreq_policy *policy = cpufreq_cpu_data[cpu];
1148         unsigned int ret_freq = 0;
1149
1150         if (!cpufreq_driver->get)
1151                 return (ret_freq);
1152
1153         ret_freq = cpufreq_driver->get(cpu);
1154
1155         if (ret_freq && policy->cur &&
1156                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1157                 /* verify no discrepancy between actual and
1158                                         saved value exists */
1159                 if (unlikely(ret_freq != policy->cur)) {
1160                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1161                         schedule_work(&policy->update);
1162                 }
1163         }
1164
1165         return (ret_freq);
1166 }
1167
1168 /**
1169  * cpufreq_get - get the current CPU frequency (in kHz)
1170  * @cpu: CPU number
1171  *
1172  * Get the CPU current (static) CPU frequency
1173  */
1174 unsigned int cpufreq_get(unsigned int cpu)
1175 {
1176         unsigned int ret_freq = 0;
1177         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1178
1179         if (!policy)
1180                 goto out;
1181
1182         if (unlikely(lock_policy_rwsem_read(cpu)))
1183                 goto out_policy;
1184
1185         ret_freq = __cpufreq_get(cpu);
1186
1187         unlock_policy_rwsem_read(cpu);
1188
1189 out_policy:
1190         cpufreq_cpu_put(policy);
1191 out:
1192         return (ret_freq);
1193 }
1194 EXPORT_SYMBOL(cpufreq_get);
1195
1196
1197 /**
1198  *      cpufreq_suspend - let the low level driver prepare for suspend
1199  */
1200
1201 static int cpufreq_suspend(struct sys_device * sysdev, pm_message_t pmsg)
1202 {
1203         int cpu = sysdev->id;
1204         int ret = 0;
1205         unsigned int cur_freq = 0;
1206         struct cpufreq_policy *cpu_policy;
1207
1208         dprintk("suspending cpu %u\n", cpu);
1209
1210         if (!cpu_online(cpu))
1211                 return 0;
1212
1213         /* we may be lax here as interrupts are off. Nonetheless
1214          * we need to grab the correct cpu policy, as to check
1215          * whether we really run on this CPU.
1216          */
1217
1218         cpu_policy = cpufreq_cpu_get(cpu);
1219         if (!cpu_policy)
1220                 return -EINVAL;
1221
1222         /* only handle each CPU group once */
1223         if (unlikely(cpu_policy->cpu != cpu)) {
1224                 cpufreq_cpu_put(cpu_policy);
1225                 return 0;
1226         }
1227
1228         if (cpufreq_driver->suspend) {
1229                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1230                 if (ret) {
1231                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1232                                         "step on CPU %u\n", cpu_policy->cpu);
1233                         cpufreq_cpu_put(cpu_policy);
1234                         return ret;
1235                 }
1236         }
1237
1238
1239         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1240                 goto out;
1241
1242         if (cpufreq_driver->get)
1243                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1244
1245         if (!cur_freq || !cpu_policy->cur) {
1246                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1247                        "frequency is what timing core thinks it is.\n");
1248                 goto out;
1249         }
1250
1251         if (unlikely(cur_freq != cpu_policy->cur)) {
1252                 struct cpufreq_freqs freqs;
1253
1254                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1255                         dprintk("Warning: CPU frequency is %u, "
1256                                "cpufreq assumed %u kHz.\n",
1257                                cur_freq, cpu_policy->cur);
1258
1259                 freqs.cpu = cpu;
1260                 freqs.old = cpu_policy->cur;
1261                 freqs.new = cur_freq;
1262
1263                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1264                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1265                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1266
1267                 cpu_policy->cur = cur_freq;
1268         }
1269
1270 out:
1271         cpufreq_cpu_put(cpu_policy);
1272         return 0;
1273 }
1274
1275 /**
1276  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1277  *
1278  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1279  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1280  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1281  *          restored.
1282  */
1283 static int cpufreq_resume(struct sys_device * sysdev)
1284 {
1285         int cpu = sysdev->id;
1286         int ret = 0;
1287         struct cpufreq_policy *cpu_policy;
1288
1289         dprintk("resuming cpu %u\n", cpu);
1290
1291         if (!cpu_online(cpu))
1292                 return 0;
1293
1294         /* we may be lax here as interrupts are off. Nonetheless
1295          * we need to grab the correct cpu policy, as to check
1296          * whether we really run on this CPU.
1297          */
1298
1299         cpu_policy = cpufreq_cpu_get(cpu);
1300         if (!cpu_policy)
1301                 return -EINVAL;
1302
1303         /* only handle each CPU group once */
1304         if (unlikely(cpu_policy->cpu != cpu)) {
1305                 cpufreq_cpu_put(cpu_policy);
1306                 return 0;
1307         }
1308
1309         if (cpufreq_driver->resume) {
1310                 ret = cpufreq_driver->resume(cpu_policy);
1311                 if (ret) {
1312                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1313                                         "step on CPU %u\n", cpu_policy->cpu);
1314                         cpufreq_cpu_put(cpu_policy);
1315                         return ret;
1316                 }
1317         }
1318
1319         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1320                 unsigned int cur_freq = 0;
1321
1322                 if (cpufreq_driver->get)
1323                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1324
1325                 if (!cur_freq || !cpu_policy->cur) {
1326                         printk(KERN_ERR "cpufreq: resume failed to assert "
1327                                         "current frequency is what timing core "
1328                                         "thinks it is.\n");
1329                         goto out;
1330                 }
1331
1332                 if (unlikely(cur_freq != cpu_policy->cur)) {
1333                         struct cpufreq_freqs freqs;
1334
1335                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1336                                 dprintk("Warning: CPU frequency "
1337                                        "is %u, cpufreq assumed %u kHz.\n",
1338                                        cur_freq, cpu_policy->cur);
1339
1340                         freqs.cpu = cpu;
1341                         freqs.old = cpu_policy->cur;
1342                         freqs.new = cur_freq;
1343
1344                         srcu_notifier_call_chain(
1345                                         &cpufreq_transition_notifier_list,
1346                                         CPUFREQ_RESUMECHANGE, &freqs);
1347                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1348
1349                         cpu_policy->cur = cur_freq;
1350                 }
1351         }
1352
1353 out:
1354         schedule_work(&cpu_policy->update);
1355         cpufreq_cpu_put(cpu_policy);
1356         return ret;
1357 }
1358
1359 static struct sysdev_driver cpufreq_sysdev_driver = {
1360         .add            = cpufreq_add_dev,
1361         .remove         = cpufreq_remove_dev,
1362         .suspend        = cpufreq_suspend,
1363         .resume         = cpufreq_resume,
1364 };
1365
1366
1367 /*********************************************************************
1368  *                     NOTIFIER LISTS INTERFACE                      *
1369  *********************************************************************/
1370
1371 /**
1372  *      cpufreq_register_notifier - register a driver with cpufreq
1373  *      @nb: notifier function to register
1374  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1375  *
1376  *      Add a driver to one of two lists: either a list of drivers that
1377  *      are notified about clock rate changes (once before and once after
1378  *      the transition), or a list of drivers that are notified about
1379  *      changes in cpufreq policy.
1380  *
1381  *      This function may sleep, and has the same return conditions as
1382  *      blocking_notifier_chain_register.
1383  */
1384 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1385 {
1386         int ret;
1387
1388         switch (list) {
1389         case CPUFREQ_TRANSITION_NOTIFIER:
1390                 ret = srcu_notifier_chain_register(
1391                                 &cpufreq_transition_notifier_list, nb);
1392                 break;
1393         case CPUFREQ_POLICY_NOTIFIER:
1394                 ret = blocking_notifier_chain_register(
1395                                 &cpufreq_policy_notifier_list, nb);
1396                 break;
1397         default:
1398                 ret = -EINVAL;
1399         }
1400
1401         return ret;
1402 }
1403 EXPORT_SYMBOL(cpufreq_register_notifier);
1404
1405
1406 /**
1407  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1408  *      @nb: notifier block to be unregistered
1409  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1410  *
1411  *      Remove a driver from the CPU frequency notifier list.
1412  *
1413  *      This function may sleep, and has the same return conditions as
1414  *      blocking_notifier_chain_unregister.
1415  */
1416 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1417 {
1418         int ret;
1419
1420         switch (list) {
1421         case CPUFREQ_TRANSITION_NOTIFIER:
1422                 ret = srcu_notifier_chain_unregister(
1423                                 &cpufreq_transition_notifier_list, nb);
1424                 break;
1425         case CPUFREQ_POLICY_NOTIFIER:
1426                 ret = blocking_notifier_chain_unregister(
1427                                 &cpufreq_policy_notifier_list, nb);
1428                 break;
1429         default:
1430                 ret = -EINVAL;
1431         }
1432
1433         return ret;
1434 }
1435 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1436
1437
1438 /*********************************************************************
1439  *                              GOVERNORS                            *
1440  *********************************************************************/
1441
1442
1443 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1444                             unsigned int target_freq,
1445                             unsigned int relation)
1446 {
1447         int retval = -EINVAL;
1448
1449         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1450                 target_freq, relation);
1451         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1452                 retval = cpufreq_driver->target(policy, target_freq, relation);
1453
1454         return retval;
1455 }
1456 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1457
1458 int cpufreq_driver_target(struct cpufreq_policy *policy,
1459                           unsigned int target_freq,
1460                           unsigned int relation)
1461 {
1462         int ret;
1463
1464         policy = cpufreq_cpu_get(policy->cpu);
1465         if (!policy)
1466                 return -EINVAL;
1467
1468         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1469                 return -EINVAL;
1470
1471         ret = __cpufreq_driver_target(policy, target_freq, relation);
1472
1473         unlock_policy_rwsem_write(policy->cpu);
1474
1475         cpufreq_cpu_put(policy);
1476         return ret;
1477 }
1478 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1479
1480 int __cpufreq_driver_getavg(struct cpufreq_policy *policy)
1481 {
1482         int ret = 0;
1483
1484         policy = cpufreq_cpu_get(policy->cpu);
1485         if (!policy)
1486                 return -EINVAL;
1487
1488         if (cpu_online(policy->cpu) && cpufreq_driver->getavg)
1489                 ret = cpufreq_driver->getavg(policy->cpu);
1490
1491         cpufreq_cpu_put(policy);
1492         return ret;
1493 }
1494 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1495
1496 /*
1497  * when "event" is CPUFREQ_GOV_LIMITS
1498  */
1499
1500 static int __cpufreq_governor(struct cpufreq_policy *policy,
1501                                         unsigned int event)
1502 {
1503         int ret;
1504
1505         /* Only must be defined when default governor is known to have latency
1506            restrictions, like e.g. conservative or ondemand.
1507            That this is the case is already ensured in Kconfig
1508         */
1509 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1510         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1511 #else
1512         struct cpufreq_governor *gov = NULL;
1513 #endif
1514
1515         if (policy->governor->max_transition_latency &&
1516             policy->cpuinfo.transition_latency >
1517             policy->governor->max_transition_latency) {
1518                 if (!gov)
1519                         return -EINVAL;
1520                 else {
1521                         printk(KERN_WARNING "%s governor failed, too long"
1522                                " transition latency of HW, fallback"
1523                                " to %s governor\n",
1524                                policy->governor->name,
1525                                gov->name);
1526                         policy->governor = gov;
1527                 }
1528         }
1529
1530         if (!try_module_get(policy->governor->owner))
1531                 return -EINVAL;
1532
1533         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1534                                                 policy->cpu, event);
1535         ret = policy->governor->governor(policy, event);
1536
1537         /* we keep one module reference alive for
1538                         each CPU governed by this CPU */
1539         if ((event != CPUFREQ_GOV_START) || ret)
1540                 module_put(policy->governor->owner);
1541         if ((event == CPUFREQ_GOV_STOP) && !ret)
1542                 module_put(policy->governor->owner);
1543
1544         return ret;
1545 }
1546
1547
1548 int cpufreq_register_governor(struct cpufreq_governor *governor)
1549 {
1550         int err;
1551
1552         if (!governor)
1553                 return -EINVAL;
1554
1555         mutex_lock(&cpufreq_governor_mutex);
1556
1557         err = -EBUSY;
1558         if (__find_governor(governor->name) == NULL) {
1559                 err = 0;
1560                 list_add(&governor->governor_list, &cpufreq_governor_list);
1561         }
1562
1563         mutex_unlock(&cpufreq_governor_mutex);
1564         return err;
1565 }
1566 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1567
1568
1569 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1570 {
1571         if (!governor)
1572                 return;
1573
1574         mutex_lock(&cpufreq_governor_mutex);
1575         list_del(&governor->governor_list);
1576         mutex_unlock(&cpufreq_governor_mutex);
1577         return;
1578 }
1579 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1580
1581
1582
1583 /*********************************************************************
1584  *                          POLICY INTERFACE                         *
1585  *********************************************************************/
1586
1587 /**
1588  * cpufreq_get_policy - get the current cpufreq_policy
1589  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1590  *
1591  * Reads the current cpufreq policy.
1592  */
1593 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1594 {
1595         struct cpufreq_policy *cpu_policy;
1596         if (!policy)
1597                 return -EINVAL;
1598
1599         cpu_policy = cpufreq_cpu_get(cpu);
1600         if (!cpu_policy)
1601                 return -EINVAL;
1602
1603         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1604
1605         cpufreq_cpu_put(cpu_policy);
1606         return 0;
1607 }
1608 EXPORT_SYMBOL(cpufreq_get_policy);
1609
1610
1611 /*
1612  * data   : current policy.
1613  * policy : policy to be set.
1614  */
1615 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1616                                 struct cpufreq_policy *policy)
1617 {
1618         int ret = 0;
1619
1620         cpufreq_debug_disable_ratelimit();
1621         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1622                 policy->min, policy->max);
1623
1624         memcpy(&policy->cpuinfo, &data->cpuinfo,
1625                                 sizeof(struct cpufreq_cpuinfo));
1626
1627         if (policy->min > data->max || policy->max < data->min) {
1628                 ret = -EINVAL;
1629                 goto error_out;
1630         }
1631
1632         /* verify the cpu speed can be set within this limit */
1633         ret = cpufreq_driver->verify(policy);
1634         if (ret)
1635                 goto error_out;
1636
1637         /* adjust if necessary - all reasons */
1638         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1639                         CPUFREQ_ADJUST, policy);
1640
1641         /* adjust if necessary - hardware incompatibility*/
1642         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1643                         CPUFREQ_INCOMPATIBLE, policy);
1644
1645         /* verify the cpu speed can be set within this limit,
1646            which might be different to the first one */
1647         ret = cpufreq_driver->verify(policy);
1648         if (ret)
1649                 goto error_out;
1650
1651         /* notification of the new policy */
1652         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1653                         CPUFREQ_NOTIFY, policy);
1654
1655         data->min = policy->min;
1656         data->max = policy->max;
1657
1658         dprintk("new min and max freqs are %u - %u kHz\n",
1659                                         data->min, data->max);
1660
1661         if (cpufreq_driver->setpolicy) {
1662                 data->policy = policy->policy;
1663                 dprintk("setting range\n");
1664                 ret = cpufreq_driver->setpolicy(policy);
1665         } else {
1666                 if (policy->governor != data->governor) {
1667                         /* save old, working values */
1668                         struct cpufreq_governor *old_gov = data->governor;
1669
1670                         dprintk("governor switch\n");
1671
1672                         /* end old governor */
1673                         if (data->governor)
1674                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1675
1676                         /* start new governor */
1677                         data->governor = policy->governor;
1678                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1679                                 /* new governor failed, so re-start old one */
1680                                 dprintk("starting governor %s failed\n",
1681                                                         data->governor->name);
1682                                 if (old_gov) {
1683                                         data->governor = old_gov;
1684                                         __cpufreq_governor(data,
1685                                                            CPUFREQ_GOV_START);
1686                                 }
1687                                 ret = -EINVAL;
1688                                 goto error_out;
1689                         }
1690                         /* might be a policy change, too, so fall through */
1691                 }
1692                 dprintk("governor: change or update limits\n");
1693                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1694         }
1695
1696 error_out:
1697         cpufreq_debug_enable_ratelimit();
1698         return ret;
1699 }
1700
1701 /**
1702  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1703  *      @cpu: CPU which shall be re-evaluated
1704  *
1705  *      Usefull for policy notifiers which have different necessities
1706  *      at different times.
1707  */
1708 int cpufreq_update_policy(unsigned int cpu)
1709 {
1710         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1711         struct cpufreq_policy policy;
1712         int ret = 0;
1713
1714         if (!data)
1715                 return -ENODEV;
1716
1717         if (unlikely(lock_policy_rwsem_write(cpu)))
1718                 return -EINVAL;
1719
1720         dprintk("updating policy for CPU %u\n", cpu);
1721         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1722         policy.min = data->user_policy.min;
1723         policy.max = data->user_policy.max;
1724         policy.policy = data->user_policy.policy;
1725         policy.governor = data->user_policy.governor;
1726
1727         /* BIOS might change freq behind our back
1728           -> ask driver for current freq and notify governors about a change */
1729         if (cpufreq_driver->get) {
1730                 policy.cur = cpufreq_driver->get(cpu);
1731                 if (!data->cur) {
1732                         dprintk("Driver did not initialize current freq");
1733                         data->cur = policy.cur;
1734                 } else {
1735                         if (data->cur != policy.cur)
1736                                 cpufreq_out_of_sync(cpu, data->cur,
1737                                                                 policy.cur);
1738                 }
1739         }
1740
1741         ret = __cpufreq_set_policy(data, &policy);
1742
1743         unlock_policy_rwsem_write(cpu);
1744
1745         cpufreq_cpu_put(data);
1746         return ret;
1747 }
1748 EXPORT_SYMBOL(cpufreq_update_policy);
1749
1750 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1751                                         unsigned long action, void *hcpu)
1752 {
1753         unsigned int cpu = (unsigned long)hcpu;
1754         struct sys_device *sys_dev;
1755
1756         sys_dev = get_cpu_sysdev(cpu);
1757         if (sys_dev) {
1758                 switch (action) {
1759                 case CPU_ONLINE:
1760                 case CPU_ONLINE_FROZEN:
1761                         cpufreq_add_dev(sys_dev);
1762                         break;
1763                 case CPU_DOWN_PREPARE:
1764                 case CPU_DOWN_PREPARE_FROZEN:
1765                         if (unlikely(lock_policy_rwsem_write(cpu)))
1766                                 BUG();
1767
1768                         __cpufreq_remove_dev(sys_dev);
1769                         break;
1770                 case CPU_DOWN_FAILED:
1771                 case CPU_DOWN_FAILED_FROZEN:
1772                         cpufreq_add_dev(sys_dev);
1773                         break;
1774                 }
1775         }
1776         return NOTIFY_OK;
1777 }
1778
1779 static struct notifier_block __cpuinitdata cpufreq_cpu_notifier =
1780 {
1781     .notifier_call = cpufreq_cpu_callback,
1782 };
1783
1784 /*********************************************************************
1785  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1786  *********************************************************************/
1787
1788 /**
1789  * cpufreq_register_driver - register a CPU Frequency driver
1790  * @driver_data: A struct cpufreq_driver containing the values#
1791  * submitted by the CPU Frequency driver.
1792  *
1793  *   Registers a CPU Frequency driver to this core code. This code
1794  * returns zero on success, -EBUSY when another driver got here first
1795  * (and isn't unregistered in the meantime).
1796  *
1797  */
1798 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1799 {
1800         unsigned long flags;
1801         int ret;
1802
1803         if (!driver_data || !driver_data->verify || !driver_data->init ||
1804             ((!driver_data->setpolicy) && (!driver_data->target)))
1805                 return -EINVAL;
1806
1807         dprintk("trying to register driver %s\n", driver_data->name);
1808
1809         if (driver_data->setpolicy)
1810                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1811
1812         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1813         if (cpufreq_driver) {
1814                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1815                 return -EBUSY;
1816         }
1817         cpufreq_driver = driver_data;
1818         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1819
1820         ret = sysdev_driver_register(&cpu_sysdev_class,&cpufreq_sysdev_driver);
1821
1822         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1823                 int i;
1824                 ret = -ENODEV;
1825
1826                 /* check for at least one working CPU */
1827                 for (i=0; i<NR_CPUS; i++)
1828                         if (cpufreq_cpu_data[i])
1829                                 ret = 0;
1830
1831                 /* if all ->init() calls failed, unregister */
1832                 if (ret) {
1833                         dprintk("no CPU initialized for driver %s\n",
1834                                                         driver_data->name);
1835                         sysdev_driver_unregister(&cpu_sysdev_class,
1836                                                 &cpufreq_sysdev_driver);
1837
1838                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1839                         cpufreq_driver = NULL;
1840                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1841                 }
1842         }
1843
1844         if (!ret) {
1845                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1846                 dprintk("driver %s up and running\n", driver_data->name);
1847                 cpufreq_debug_enable_ratelimit();
1848         }
1849
1850         return (ret);
1851 }
1852 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1853
1854
1855 /**
1856  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1857  *
1858  *    Unregister the current CPUFreq driver. Only call this if you have
1859  * the right to do so, i.e. if you have succeeded in initialising before!
1860  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1861  * currently not initialised.
1862  */
1863 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1864 {
1865         unsigned long flags;
1866
1867         cpufreq_debug_disable_ratelimit();
1868
1869         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1870                 cpufreq_debug_enable_ratelimit();
1871                 return -EINVAL;
1872         }
1873
1874         dprintk("unregistering driver %s\n", driver->name);
1875
1876         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1877         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1878
1879         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1880         cpufreq_driver = NULL;
1881         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1882
1883         return 0;
1884 }
1885 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1886
1887 static int __init cpufreq_core_init(void)
1888 {
1889         int cpu;
1890
1891         for_each_possible_cpu(cpu) {
1892                 per_cpu(policy_cpu, cpu) = -1;
1893                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1894         }
1895         return 0;
1896 }
1897
1898 core_initcall(cpufreq_core_init);