Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux
[linux-drm-fsl-dcu.git] / drivers / idle / intel_idle.c
1 /*
2  * intel_idle.c - native hardware idle loop for modern Intel processors
3  *
4  * Copyright (c) 2010, Intel Corporation.
5  * Len Brown <len.brown@intel.com>
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms and conditions of the GNU General Public License,
9  * version 2, as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14  * more details.
15  *
16  * You should have received a copy of the GNU General Public License along with
17  * this program; if not, write to the Free Software Foundation, Inc.,
18  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19  */
20
21 /*
22  * intel_idle is a cpuidle driver that loads on specific Intel processors
23  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
24  * make Linux more efficient on these processors, as intel_idle knows
25  * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
26  */
27
28 /*
29  * Design Assumptions
30  *
31  * All CPUs have same idle states as boot CPU
32  *
33  * Chipset BM_STS (bus master status) bit is a NOP
34  *      for preventing entry into deep C-stats
35  */
36
37 /*
38  * Known limitations
39  *
40  * The driver currently initializes for_each_online_cpu() upon modprobe.
41  * It it unaware of subsequent processors hot-added to the system.
42  * This means that if you boot with maxcpus=n and later online
43  * processors above n, those processors will use C1 only.
44  *
45  * ACPI has a .suspend hack to turn off deep c-statees during suspend
46  * to avoid complications with the lapic timer workaround.
47  * Have not seen issues with suspend, but may need same workaround here.
48  *
49  * There is currently no kernel-based automatic probing/loading mechanism
50  * if the driver is built as a module.
51  */
52
53 /* un-comment DEBUG to enable pr_debug() statements */
54 #define DEBUG
55
56 #include <linux/kernel.h>
57 #include <linux/cpuidle.h>
58 #include <linux/clockchips.h>
59 #include <linux/hrtimer.h>      /* ktime_get_real() */
60 #include <trace/events/power.h>
61 #include <linux/sched.h>
62 #include <linux/notifier.h>
63 #include <linux/cpu.h>
64 #include <linux/module.h>
65 #include <asm/cpu_device_id.h>
66 #include <asm/mwait.h>
67 #include <asm/msr.h>
68
69 #define INTEL_IDLE_VERSION "0.4"
70 #define PREFIX "intel_idle: "
71
72 static struct cpuidle_driver intel_idle_driver = {
73         .name = "intel_idle",
74         .owner = THIS_MODULE,
75 };
76 /* intel_idle.max_cstate=0 disables driver */
77 static int max_cstate = MWAIT_MAX_NUM_CSTATES - 1;
78
79 static unsigned int mwait_substates;
80
81 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
82 /* Reliable LAPIC Timer States, bit 1 for C1 etc.  */
83 static unsigned int lapic_timer_reliable_states = (1 << 1);      /* Default to only C1 */
84
85 struct idle_cpu {
86         struct cpuidle_state *state_table;
87
88         /*
89          * Hardware C-state auto-demotion may not always be optimal.
90          * Indicate which enable bits to clear here.
91          */
92         unsigned long auto_demotion_disable_flags;
93 };
94
95 static const struct idle_cpu *icpu;
96 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
97 static int intel_idle(struct cpuidle_device *dev,
98                         struct cpuidle_driver *drv, int index);
99 static int intel_idle_cpu_init(int cpu);
100
101 static struct cpuidle_state *cpuidle_state_table;
102
103 /*
104  * Set this flag for states where the HW flushes the TLB for us
105  * and so we don't need cross-calls to keep it consistent.
106  * If this flag is set, SW flushes the TLB, so even if the
107  * HW doesn't do the flushing, this flag is safe to use.
108  */
109 #define CPUIDLE_FLAG_TLB_FLUSHED        0x10000
110
111 /*
112  * States are indexed by the cstate number,
113  * which is also the index into the MWAIT hint array.
114  * Thus C0 is a dummy.
115  */
116 static struct cpuidle_state nehalem_cstates[MWAIT_MAX_NUM_CSTATES] = {
117         { /* MWAIT C0 */ },
118         { /* MWAIT C1 */
119                 .name = "C1-NHM",
120                 .desc = "MWAIT 0x00",
121                 .flags = CPUIDLE_FLAG_TIME_VALID,
122                 .exit_latency = 3,
123                 .target_residency = 6,
124                 .enter = &intel_idle },
125         { /* MWAIT C2 */
126                 .name = "C3-NHM",
127                 .desc = "MWAIT 0x10",
128                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
129                 .exit_latency = 20,
130                 .target_residency = 80,
131                 .enter = &intel_idle },
132         { /* MWAIT C3 */
133                 .name = "C6-NHM",
134                 .desc = "MWAIT 0x20",
135                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
136                 .exit_latency = 200,
137                 .target_residency = 800,
138                 .enter = &intel_idle },
139 };
140
141 static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = {
142         { /* MWAIT C0 */ },
143         { /* MWAIT C1 */
144                 .name = "C1-SNB",
145                 .desc = "MWAIT 0x00",
146                 .flags = CPUIDLE_FLAG_TIME_VALID,
147                 .exit_latency = 1,
148                 .target_residency = 1,
149                 .enter = &intel_idle },
150         { /* MWAIT C2 */
151                 .name = "C3-SNB",
152                 .desc = "MWAIT 0x10",
153                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
154                 .exit_latency = 80,
155                 .target_residency = 211,
156                 .enter = &intel_idle },
157         { /* MWAIT C3 */
158                 .name = "C6-SNB",
159                 .desc = "MWAIT 0x20",
160                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
161                 .exit_latency = 104,
162                 .target_residency = 345,
163                 .enter = &intel_idle },
164         { /* MWAIT C4 */
165                 .name = "C7-SNB",
166                 .desc = "MWAIT 0x30",
167                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
168                 .exit_latency = 109,
169                 .target_residency = 345,
170                 .enter = &intel_idle },
171 };
172
173 static struct cpuidle_state ivb_cstates[MWAIT_MAX_NUM_CSTATES] = {
174         { /* MWAIT C0 */ },
175         { /* MWAIT C1 */
176                 .name = "C1-IVB",
177                 .desc = "MWAIT 0x00",
178                 .flags = CPUIDLE_FLAG_TIME_VALID,
179                 .exit_latency = 1,
180                 .target_residency = 1,
181                 .enter = &intel_idle },
182         { /* MWAIT C2 */
183                 .name = "C3-IVB",
184                 .desc = "MWAIT 0x10",
185                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
186                 .exit_latency = 59,
187                 .target_residency = 156,
188                 .enter = &intel_idle },
189         { /* MWAIT C3 */
190                 .name = "C6-IVB",
191                 .desc = "MWAIT 0x20",
192                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
193                 .exit_latency = 80,
194                 .target_residency = 300,
195                 .enter = &intel_idle },
196         { /* MWAIT C4 */
197                 .name = "C7-IVB",
198                 .desc = "MWAIT 0x30",
199                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
200                 .exit_latency = 87,
201                 .target_residency = 300,
202                 .enter = &intel_idle },
203 };
204
205 static struct cpuidle_state atom_cstates[MWAIT_MAX_NUM_CSTATES] = {
206         { /* MWAIT C0 */ },
207         { /* MWAIT C1 */
208                 .name = "C1-ATM",
209                 .desc = "MWAIT 0x00",
210                 .flags = CPUIDLE_FLAG_TIME_VALID,
211                 .exit_latency = 1,
212                 .target_residency = 4,
213                 .enter = &intel_idle },
214         { /* MWAIT C2 */
215                 .name = "C2-ATM",
216                 .desc = "MWAIT 0x10",
217                 .flags = CPUIDLE_FLAG_TIME_VALID,
218                 .exit_latency = 20,
219                 .target_residency = 80,
220                 .enter = &intel_idle },
221         { /* MWAIT C3 */ },
222         { /* MWAIT C4 */
223                 .name = "C4-ATM",
224                 .desc = "MWAIT 0x30",
225                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
226                 .exit_latency = 100,
227                 .target_residency = 400,
228                 .enter = &intel_idle },
229         { /* MWAIT C5 */ },
230         { /* MWAIT C6 */
231                 .name = "C6-ATM",
232                 .desc = "MWAIT 0x52",
233                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
234                 .exit_latency = 140,
235                 .target_residency = 560,
236                 .enter = &intel_idle },
237 };
238
239 static long get_driver_data(int cstate)
240 {
241         int driver_data;
242         switch (cstate) {
243
244         case 1: /* MWAIT C1 */
245                 driver_data = 0x00;
246                 break;
247         case 2: /* MWAIT C2 */
248                 driver_data = 0x10;
249                 break;
250         case 3: /* MWAIT C3 */
251                 driver_data = 0x20;
252                 break;
253         case 4: /* MWAIT C4 */
254                 driver_data = 0x30;
255                 break;
256         case 5: /* MWAIT C5 */
257                 driver_data = 0x40;
258                 break;
259         case 6: /* MWAIT C6 */
260                 driver_data = 0x52;
261                 break;
262         default:
263                 driver_data = 0x00;
264         }
265         return driver_data;
266 }
267
268 /**
269  * intel_idle
270  * @dev: cpuidle_device
271  * @drv: cpuidle driver
272  * @index: index of cpuidle state
273  *
274  * Must be called under local_irq_disable().
275  */
276 static int intel_idle(struct cpuidle_device *dev,
277                 struct cpuidle_driver *drv, int index)
278 {
279         unsigned long ecx = 1; /* break on interrupt flag */
280         struct cpuidle_state *state = &drv->states[index];
281         struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
282         unsigned long eax = (unsigned long)cpuidle_get_statedata(state_usage);
283         unsigned int cstate;
284         ktime_t kt_before, kt_after;
285         s64 usec_delta;
286         int cpu = smp_processor_id();
287
288         cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
289
290         /*
291          * leave_mm() to avoid costly and often unnecessary wakeups
292          * for flushing the user TLB's associated with the active mm.
293          */
294         if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
295                 leave_mm(cpu);
296
297         if (!(lapic_timer_reliable_states & (1 << (cstate))))
298                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
299
300         kt_before = ktime_get_real();
301
302         stop_critical_timings();
303         if (!need_resched()) {
304
305                 __monitor((void *)&current_thread_info()->flags, 0, 0);
306                 smp_mb();
307                 if (!need_resched())
308                         __mwait(eax, ecx);
309         }
310
311         start_critical_timings();
312
313         kt_after = ktime_get_real();
314         usec_delta = ktime_to_us(ktime_sub(kt_after, kt_before));
315
316         local_irq_enable();
317
318         if (!(lapic_timer_reliable_states & (1 << (cstate))))
319                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
320
321         /* Update cpuidle counters */
322         dev->last_residency = (int)usec_delta;
323
324         return index;
325 }
326
327 static void __setup_broadcast_timer(void *arg)
328 {
329         unsigned long reason = (unsigned long)arg;
330         int cpu = smp_processor_id();
331
332         reason = reason ?
333                 CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF;
334
335         clockevents_notify(reason, &cpu);
336 }
337
338 static int cpu_hotplug_notify(struct notifier_block *n,
339                               unsigned long action, void *hcpu)
340 {
341         int hotcpu = (unsigned long)hcpu;
342         struct cpuidle_device *dev;
343
344         switch (action & 0xf) {
345         case CPU_ONLINE:
346
347                 if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
348                         smp_call_function_single(hotcpu, __setup_broadcast_timer,
349                                                  (void *)true, 1);
350
351                 /*
352                  * Some systems can hotplug a cpu at runtime after
353                  * the kernel has booted, we have to initialize the
354                  * driver in this case
355                  */
356                 dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu);
357                 if (!dev->registered)
358                         intel_idle_cpu_init(hotcpu);
359
360                 break;
361         }
362         return NOTIFY_OK;
363 }
364
365 static struct notifier_block cpu_hotplug_notifier = {
366         .notifier_call = cpu_hotplug_notify,
367 };
368
369 static void auto_demotion_disable(void *dummy)
370 {
371         unsigned long long msr_bits;
372
373         rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
374         msr_bits &= ~(icpu->auto_demotion_disable_flags);
375         wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
376 }
377
378 static const struct idle_cpu idle_cpu_nehalem = {
379         .state_table = nehalem_cstates,
380         .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
381 };
382
383 static const struct idle_cpu idle_cpu_atom = {
384         .state_table = atom_cstates,
385 };
386
387 static const struct idle_cpu idle_cpu_lincroft = {
388         .state_table = atom_cstates,
389         .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
390 };
391
392 static const struct idle_cpu idle_cpu_snb = {
393         .state_table = snb_cstates,
394 };
395
396 static const struct idle_cpu idle_cpu_ivb = {
397         .state_table = ivb_cstates,
398 };
399
400 #define ICPU(model, cpu) \
401         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu }
402
403 static const struct x86_cpu_id intel_idle_ids[] = {
404         ICPU(0x1a, idle_cpu_nehalem),
405         ICPU(0x1e, idle_cpu_nehalem),
406         ICPU(0x1f, idle_cpu_nehalem),
407         ICPU(0x25, idle_cpu_nehalem),
408         ICPU(0x2c, idle_cpu_nehalem),
409         ICPU(0x2e, idle_cpu_nehalem),
410         ICPU(0x1c, idle_cpu_atom),
411         ICPU(0x26, idle_cpu_lincroft),
412         ICPU(0x2f, idle_cpu_nehalem),
413         ICPU(0x2a, idle_cpu_snb),
414         ICPU(0x2d, idle_cpu_snb),
415         ICPU(0x3a, idle_cpu_ivb),
416         {}
417 };
418 MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids);
419
420 /*
421  * intel_idle_probe()
422  */
423 static int intel_idle_probe(void)
424 {
425         unsigned int eax, ebx, ecx;
426         const struct x86_cpu_id *id;
427
428         if (max_cstate == 0) {
429                 pr_debug(PREFIX "disabled\n");
430                 return -EPERM;
431         }
432
433         id = x86_match_cpu(intel_idle_ids);
434         if (!id) {
435                 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
436                     boot_cpu_data.x86 == 6)
437                         pr_debug(PREFIX "does not run on family %d model %d\n",
438                                 boot_cpu_data.x86, boot_cpu_data.x86_model);
439                 return -ENODEV;
440         }
441
442         if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
443                 return -ENODEV;
444
445         cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
446
447         if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
448             !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
449             !mwait_substates)
450                         return -ENODEV;
451
452         pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates);
453
454         icpu = (const struct idle_cpu *)id->driver_data;
455         cpuidle_state_table = icpu->state_table;
456
457         if (boot_cpu_has(X86_FEATURE_ARAT))     /* Always Reliable APIC Timer */
458                 lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
459         else
460                 on_each_cpu(__setup_broadcast_timer, (void *)true, 1);
461
462         register_cpu_notifier(&cpu_hotplug_notifier);
463
464         pr_debug(PREFIX "v" INTEL_IDLE_VERSION
465                 " model 0x%X\n", boot_cpu_data.x86_model);
466
467         pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n",
468                 lapic_timer_reliable_states);
469         return 0;
470 }
471
472 /*
473  * intel_idle_cpuidle_devices_uninit()
474  * unregister, free cpuidle_devices
475  */
476 static void intel_idle_cpuidle_devices_uninit(void)
477 {
478         int i;
479         struct cpuidle_device *dev;
480
481         for_each_online_cpu(i) {
482                 dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
483                 cpuidle_unregister_device(dev);
484         }
485
486         free_percpu(intel_idle_cpuidle_devices);
487         return;
488 }
489 /*
490  * intel_idle_cpuidle_driver_init()
491  * allocate, initialize cpuidle_states
492  */
493 static int intel_idle_cpuidle_driver_init(void)
494 {
495         int cstate;
496         struct cpuidle_driver *drv = &intel_idle_driver;
497
498         drv->state_count = 1;
499
500         for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
501                 int num_substates;
502
503                 if (cstate > max_cstate) {
504                         printk(PREFIX "max_cstate %d reached\n",
505                                 max_cstate);
506                         break;
507                 }
508
509                 /* does the state exist in CPUID.MWAIT? */
510                 num_substates = (mwait_substates >> ((cstate) * 4))
511                                         & MWAIT_SUBSTATE_MASK;
512                 if (num_substates == 0)
513                         continue;
514                 /* is the state not enabled? */
515                 if (cpuidle_state_table[cstate].enter == NULL) {
516                         /* does the driver not know about the state? */
517                         if (*cpuidle_state_table[cstate].name == '\0')
518                                 pr_debug(PREFIX "unaware of model 0x%x"
519                                         " MWAIT %d please"
520                                         " contact lenb@kernel.org",
521                                 boot_cpu_data.x86_model, cstate);
522                         continue;
523                 }
524
525                 if ((cstate > 2) &&
526                         !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
527                         mark_tsc_unstable("TSC halts in idle"
528                                         " states deeper than C2");
529
530                 drv->states[drv->state_count] = /* structure copy */
531                         cpuidle_state_table[cstate];
532
533                 drv->state_count += 1;
534         }
535
536         if (icpu->auto_demotion_disable_flags)
537                 on_each_cpu(auto_demotion_disable, NULL, 1);
538
539         return 0;
540 }
541
542
543 /*
544  * intel_idle_cpu_init()
545  * allocate, initialize, register cpuidle_devices
546  * @cpu: cpu/core to initialize
547  */
548 static int intel_idle_cpu_init(int cpu)
549 {
550         int cstate;
551         struct cpuidle_device *dev;
552
553         dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
554
555         dev->state_count = 1;
556
557         for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
558                 int num_substates;
559
560                 if (cstate > max_cstate) {
561                         printk(PREFIX "max_cstate %d reached\n", max_cstate);
562                         break;
563                 }
564
565                 /* does the state exist in CPUID.MWAIT? */
566                 num_substates = (mwait_substates >> ((cstate) * 4))
567                         & MWAIT_SUBSTATE_MASK;
568                 if (num_substates == 0)
569                         continue;
570                 /* is the state not enabled? */
571                 if (cpuidle_state_table[cstate].enter == NULL)
572                         continue;
573
574                 dev->states_usage[dev->state_count].driver_data =
575                         (void *)get_driver_data(cstate);
576
577                 dev->state_count += 1;
578         }
579
580         dev->cpu = cpu;
581
582         if (cpuidle_register_device(dev)) {
583                 pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu);
584                 intel_idle_cpuidle_devices_uninit();
585                 return -EIO;
586         }
587
588         if (icpu->auto_demotion_disable_flags)
589                 smp_call_function_single(cpu, auto_demotion_disable, NULL, 1);
590
591         return 0;
592 }
593
594 static int __init intel_idle_init(void)
595 {
596         int retval, i;
597
598         /* Do not load intel_idle at all for now if idle= is passed */
599         if (boot_option_idle_override != IDLE_NO_OVERRIDE)
600                 return -ENODEV;
601
602         retval = intel_idle_probe();
603         if (retval)
604                 return retval;
605
606         intel_idle_cpuidle_driver_init();
607         retval = cpuidle_register_driver(&intel_idle_driver);
608         if (retval) {
609                 printk(KERN_DEBUG PREFIX "intel_idle yielding to %s",
610                         cpuidle_get_driver()->name);
611                 return retval;
612         }
613
614         intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
615         if (intel_idle_cpuidle_devices == NULL)
616                 return -ENOMEM;
617
618         for_each_online_cpu(i) {
619                 retval = intel_idle_cpu_init(i);
620                 if (retval) {
621                         cpuidle_unregister_driver(&intel_idle_driver);
622                         return retval;
623                 }
624         }
625
626         return 0;
627 }
628
629 static void __exit intel_idle_exit(void)
630 {
631         intel_idle_cpuidle_devices_uninit();
632         cpuidle_unregister_driver(&intel_idle_driver);
633
634
635         if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
636                 on_each_cpu(__setup_broadcast_timer, (void *)false, 1);
637         unregister_cpu_notifier(&cpu_hotplug_notifier);
638
639         return;
640 }
641
642 module_init(intel_idle_init);
643 module_exit(intel_idle_exit);
644
645 module_param(max_cstate, int, 0444);
646
647 MODULE_AUTHOR("Len Brown <len.brown@intel.com>");
648 MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION);
649 MODULE_LICENSE("GPL");