arch/s390/kernel/perf_cpum_cf.c

   1 /*
   2  * Performance event support for s390x - CPU-measurement Counter Facility
   3  *
   4  *  Copyright IBM Corp. 2012
   5  *  Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License (version 2 only)
   9  * as published by the Free Software Foundation.
  10  */
  11 #define KMSG_COMPONENT  "cpum_cf"
  12 #define pr_fmt(fmt)     KMSG_COMPONENT ": " fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/kernel_stat.h>
  16 #include <linux/perf_event.h>
  17 #include <linux/percpu.h>
  18 #include <linux/notifier.h>
  19 #include <linux/init.h>
  20 #include <linux/export.h>
  21 #include <asm/ctl_reg.h>
  22 #include <asm/irq.h>
  23 #include <asm/cpu_mf.h>
  24
  25 /* CPU-measurement counter facility supports these CPU counter sets:
  26  * For CPU counter sets:
  27  *    Basic counter set:             0-31
  28  *    Problem-state counter set:    32-63
  29  *    Crypto-activity counter set:  64-127
  30  *    Extented counter set:        128-159
  31  */
  32 enum cpumf_ctr_set {
  33         /* CPU counter sets */
  34         CPUMF_CTR_SET_BASIC   = 0,
  35         CPUMF_CTR_SET_USER    = 1,
  36         CPUMF_CTR_SET_CRYPTO  = 2,
  37         CPUMF_CTR_SET_EXT     = 3,
  38
  39         /* Maximum number of counter sets */
  40         CPUMF_CTR_SET_MAX,
  41 };
  42
  43 #define CPUMF_LCCTL_ENABLE_SHIFT    16
  44 #define CPUMF_LCCTL_ACTCTL_SHIFT     0
  45 static const u64 cpumf_state_ctl[CPUMF_CTR_SET_MAX] = {
  46         [CPUMF_CTR_SET_BASIC]   = 0x02,
  47         [CPUMF_CTR_SET_USER]    = 0x04,
  48         [CPUMF_CTR_SET_CRYPTO]  = 0x08,
  49         [CPUMF_CTR_SET_EXT]     = 0x01,
  50 };
  51
  52 static void ctr_set_enable(u64 *state, int ctr_set)
  53 {
  54         *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT;
  55 }
  56 static void ctr_set_disable(u64 *state, int ctr_set)
  57 {
  58         *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT);
  59 }
  60 static void ctr_set_start(u64 *state, int ctr_set)
  61 {
  62         *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT;
  63 }
  64 static void ctr_set_stop(u64 *state, int ctr_set)
  65 {
  66         *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT);
  67 }
  68
  69 /* Local CPUMF event structure */
  70 struct cpu_hw_events {
  71         struct cpumf_ctr_info   info;
  72         atomic_t                ctr_set[CPUMF_CTR_SET_MAX];
  73         u64                     state, tx_state;
  74         unsigned int            flags;
  75 };
  76 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  77         .ctr_set = {
  78                 [CPUMF_CTR_SET_BASIC]  = ATOMIC_INIT(0),
  79                 [CPUMF_CTR_SET_USER]   = ATOMIC_INIT(0),
  80                 [CPUMF_CTR_SET_CRYPTO] = ATOMIC_INIT(0),
  81                 [CPUMF_CTR_SET_EXT]    = ATOMIC_INIT(0),
  82         },
  83         .state = 0,
  84         .flags = 0,
  85 };
  86
  87 static int get_counter_set(u64 event)
  88 {
  89         int set = -1;
  90
  91         if (event < 32)
  92                 set = CPUMF_CTR_SET_BASIC;
  93         else if (event < 64)
  94                 set = CPUMF_CTR_SET_USER;
  95         else if (event < 128)
  96                 set = CPUMF_CTR_SET_CRYPTO;
  97         else if (event < 256)
  98                 set = CPUMF_CTR_SET_EXT;
  99
 100         return set;
 101 }
 102
 103 static int validate_event(const struct hw_perf_event *hwc)
 104 {
 105         switch (hwc->config_base) {
 106         case CPUMF_CTR_SET_BASIC:
 107         case CPUMF_CTR_SET_USER:
 108         case CPUMF_CTR_SET_CRYPTO:
 109         case CPUMF_CTR_SET_EXT:
 110                 /* check for reserved counters */
 111                 if ((hwc->config >=  6 && hwc->config <=  31) ||
 112                     (hwc->config >= 38 && hwc->config <=  63) ||
 113                     (hwc->config >= 80 && hwc->config <= 127))
 114                         return -EOPNOTSUPP;
 115                 break;
 116         default:
 117                 return -EINVAL;
 118         }
 119
 120         return 0;
 121 }
 122
 123 static int validate_ctr_version(const struct hw_perf_event *hwc)
 124 {
 125         struct cpu_hw_events *cpuhw;
 126         int err = 0;
 127
 128         cpuhw = &get_cpu_var(cpu_hw_events);
 129
 130         /* check required version for counter sets */
 131         switch (hwc->config_base) {
 132         case CPUMF_CTR_SET_BASIC:
 133         case CPUMF_CTR_SET_USER:
 134                 if (cpuhw->info.cfvn < 1)
 135                         err = -EOPNOTSUPP;
 136                 break;
 137         case CPUMF_CTR_SET_CRYPTO:
 138         case CPUMF_CTR_SET_EXT:
 139                 if (cpuhw->info.csvn < 1)
 140                         err = -EOPNOTSUPP;
 141                 if ((cpuhw->info.csvn == 1 && hwc->config > 159) ||
 142                     (cpuhw->info.csvn == 2 && hwc->config > 175) ||
 143                     (cpuhw->info.csvn  > 2 && hwc->config > 255))
 144                         err = -EOPNOTSUPP;
 145                 break;
 146         }
 147
 148         put_cpu_var(cpu_hw_events);
 149         return err;
 150 }
 151
 152 static int validate_ctr_auth(const struct hw_perf_event *hwc)
 153 {
 154         struct cpu_hw_events *cpuhw;
 155         u64 ctrs_state;
 156         int err = 0;
 157
 158         cpuhw = &get_cpu_var(cpu_hw_events);
 159
 160         /* Check authorization for cpu counter sets.
 161          * If the particular CPU counter set is not authorized,
 162          * return with -ENOENT in order to fall back to other
 163          * PMUs that might suffice the event request.
 164          */
 165         ctrs_state = cpumf_state_ctl[hwc->config_base];
 166         if (!(ctrs_state & cpuhw->info.auth_ctl))
 167                 err = -ENOENT;
 168
 169         put_cpu_var(cpu_hw_events);
 170         return err;
 171 }
 172
 173 /*
 174  * Change the CPUMF state to active.
 175  * Enable and activate the CPU-counter sets according
 176  * to the per-cpu control state.
 177  */
 178 static void cpumf_pmu_enable(struct pmu *pmu)
 179 {
 180         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 181         int err;
 182
 183         if (cpuhw->flags & PMU_F_ENABLED)
 184                 return;
 185
 186         err = lcctl(cpuhw->state);
 187         if (err) {
 188                 pr_err("Enabling the performance measuring unit "
 189                        "failed with rc=%x\n", err);
 190                 return;
 191         }
 192
 193         cpuhw->flags |= PMU_F_ENABLED;
 194 }
 195
 196 /*
 197  * Change the CPUMF state to inactive.
 198  * Disable and enable (inactive) the CPU-counter sets according
 199  * to the per-cpu control state.
 200  */
 201 static void cpumf_pmu_disable(struct pmu *pmu)
 202 {
 203         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 204         int err;
 205         u64 inactive;
 206
 207         if (!(cpuhw->flags & PMU_F_ENABLED))
 208                 return;
 209
 210         inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
 211         err = lcctl(inactive);
 212         if (err) {
 213                 pr_err("Disabling the performance measuring unit "
 214                        "failed with rc=%x\n", err);
 215                 return;
 216         }
 217
 218         cpuhw->flags &= ~PMU_F_ENABLED;
 219 }
 220
 221
 222 /* Number of perf events counting hardware events */
 223 static atomic_t num_events = ATOMIC_INIT(0);
 224 /* Used to avoid races in calling reserve/release_cpumf_hardware */
 225 static DEFINE_MUTEX(pmc_reserve_mutex);
 226
 227 /* CPU-measurement alerts for the counter facility */
 228 static void cpumf_measurement_alert(struct ext_code ext_code,
 229                                     unsigned int alert, unsigned long unused)
 230 {
 231         struct cpu_hw_events *cpuhw;
 232
 233         if (!(alert & CPU_MF_INT_CF_MASK))
 234                 return;
 235
 236         inc_irq_stat(IRQEXT_CMC);
 237         cpuhw = this_cpu_ptr(&cpu_hw_events);
 238
 239         /* Measurement alerts are shared and might happen when the PMU
 240          * is not reserved.  Ignore these alerts in this case. */
 241         if (!(cpuhw->flags & PMU_F_RESERVED))
 242                 return;
 243
 244         /* counter authorization change alert */
 245         if (alert & CPU_MF_INT_CF_CACA)
 246                 qctri(&cpuhw->info);
 247
 248         /* loss of counter data alert */
 249         if (alert & CPU_MF_INT_CF_LCDA)
 250                 pr_err("CPU[%i] Counter data was lost\n", smp_processor_id());
 251 }
 252
 253 #define PMC_INIT      0
 254 #define PMC_RELEASE   1
 255 static void setup_pmc_cpu(void *flags)
 256 {
 257         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 258
 259         switch (*((int *) flags)) {
 260         case PMC_INIT:
 261                 memset(&cpuhw->info, 0, sizeof(cpuhw->info));
 262                 qctri(&cpuhw->info);
 263                 cpuhw->flags |= PMU_F_RESERVED;
 264                 break;
 265
 266         case PMC_RELEASE:
 267                 cpuhw->flags &= ~PMU_F_RESERVED;
 268                 break;
 269         }
 270
 271         /* Disable CPU counter sets */
 272         lcctl(0);
 273 }
 274
 275 /* Initialize the CPU-measurement facility */
 276 static int reserve_pmc_hardware(void)
 277 {
 278         int flags = PMC_INIT;
 279
 280         on_each_cpu(setup_pmc_cpu, &flags, 1);
 281         irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 282
 283         return 0;
 284 }
 285
 286 /* Release the CPU-measurement facility */
 287 static void release_pmc_hardware(void)
 288 {
 289         int flags = PMC_RELEASE;
 290
 291         on_each_cpu(setup_pmc_cpu, &flags, 1);
 292         irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 293 }
 294
 295 /* Release the PMU if event is the last perf event */
 296 static void hw_perf_event_destroy(struct perf_event *event)
 297 {
 298         if (!atomic_add_unless(&num_events, -1, 1)) {
 299                 mutex_lock(&pmc_reserve_mutex);
 300                 if (atomic_dec_return(&num_events) == 0)
 301                         release_pmc_hardware();
 302                 mutex_unlock(&pmc_reserve_mutex);
 303         }
 304 }
 305
 306 /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */
 307 static const int cpumf_generic_events_basic[] = {
 308         [PERF_COUNT_HW_CPU_CYCLES]          = 0,
 309         [PERF_COUNT_HW_INSTRUCTIONS]        = 1,
 310         [PERF_COUNT_HW_CACHE_REFERENCES]    = -1,
 311         [PERF_COUNT_HW_CACHE_MISSES]        = -1,
 312         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
 313         [PERF_COUNT_HW_BRANCH_MISSES]       = -1,
 314         [PERF_COUNT_HW_BUS_CYCLES]          = -1,
 315 };
 316 /* CPUMF <-> perf event mappings for userspace (problem-state set) */
 317 static const int cpumf_generic_events_user[] = {
 318         [PERF_COUNT_HW_CPU_CYCLES]          = 32,
 319         [PERF_COUNT_HW_INSTRUCTIONS]        = 33,
 320         [PERF_COUNT_HW_CACHE_REFERENCES]    = -1,
 321         [PERF_COUNT_HW_CACHE_MISSES]        = -1,
 322         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
 323         [PERF_COUNT_HW_BRANCH_MISSES]       = -1,
 324         [PERF_COUNT_HW_BUS_CYCLES]          = -1,
 325 };
 326
 327 static int __hw_perf_event_init(struct perf_event *event)
 328 {
 329         struct perf_event_attr *attr = &event->attr;
 330         struct hw_perf_event *hwc = &event->hw;
 331         int err;
 332         u64 ev;
 333
 334         switch (attr->type) {
 335         case PERF_TYPE_RAW:
 336                 /* Raw events are used to access counters directly,
 337                  * hence do not permit excludes */
 338                 if (attr->exclude_kernel || attr->exclude_user ||
 339                     attr->exclude_hv)
 340                         return -EOPNOTSUPP;
 341                 ev = attr->config;
 342                 break;
 343
 344         case PERF_TYPE_HARDWARE:
 345                 ev = attr->config;
 346                 /* Count user space (problem-state) only */
 347                 if (!attr->exclude_user && attr->exclude_kernel) {
 348                         if (ev >= ARRAY_SIZE(cpumf_generic_events_user))
 349                                 return -EOPNOTSUPP;
 350                         ev = cpumf_generic_events_user[ev];
 351
 352                 /* No support for kernel space counters only */
 353                 } else if (!attr->exclude_kernel && attr->exclude_user) {
 354                         return -EOPNOTSUPP;
 355
 356                 /* Count user and kernel space */
 357                 } else {
 358                         if (ev >= ARRAY_SIZE(cpumf_generic_events_basic))
 359                                 return -EOPNOTSUPP;
 360                         ev = cpumf_generic_events_basic[ev];
 361                 }
 362                 break;
 363
 364         default:
 365                 return -ENOENT;
 366         }
 367
 368         if (ev == -1)
 369                 return -ENOENT;
 370
 371         if (ev >= PERF_CPUM_CF_MAX_CTR)
 372                 return -EINVAL;
 373
 374         /* Use the hardware perf event structure to store the counter number
 375          * in 'config' member and the counter set to which the counter belongs
 376          * in the 'config_base'.  The counter set (config_base) is then used
 377          * to enable/disable the counters.
 378          */
 379         hwc->config = ev;
 380         hwc->config_base = get_counter_set(ev);
 381
 382         /* Validate the counter that is assigned to this event.
 383          * Because the counter facility can use numerous counters at the
 384          * same time without constraints, it is not necessary to explicity
 385          * validate event groups (event->group_leader != event).
 386          */
 387         err = validate_event(hwc);
 388         if (err)
 389                 return err;
 390
 391         /* Initialize for using the CPU-measurement counter facility */
 392         if (!atomic_inc_not_zero(&num_events)) {
 393                 mutex_lock(&pmc_reserve_mutex);
 394                 if (atomic_read(&num_events) == 0 && reserve_pmc_hardware())
 395                         err = -EBUSY;
 396                 else
 397                         atomic_inc(&num_events);
 398                 mutex_unlock(&pmc_reserve_mutex);
 399         }
 400         event->destroy = hw_perf_event_destroy;
 401
 402         /* Finally, validate version and authorization of the counter set */
 403         err = validate_ctr_auth(hwc);
 404         if (!err)
 405                 err = validate_ctr_version(hwc);
 406
 407         return err;
 408 }
 409
 410 static int cpumf_pmu_event_init(struct perf_event *event)
 411 {
 412         int err;
 413
 414         switch (event->attr.type) {
 415         case PERF_TYPE_HARDWARE:
 416         case PERF_TYPE_HW_CACHE:
 417         case PERF_TYPE_RAW:
 418                 err = __hw_perf_event_init(event);
 419                 break;
 420         default:
 421                 return -ENOENT;
 422         }
 423
 424         if (unlikely(err) && event->destroy)
 425                 event->destroy(event);
 426
 427         return err;
 428 }
 429
 430 static int hw_perf_event_reset(struct perf_event *event)
 431 {
 432         u64 prev, new;
 433         int err;
 434
 435         do {
 436                 prev = local64_read(&event->hw.prev_count);
 437                 err = ecctr(event->hw.config, &new);
 438                 if (err) {
 439                         if (err != 3)
 440                                 break;
 441                         /* The counter is not (yet) available. This
 442                          * might happen if the counter set to which
 443                          * this counter belongs is in the disabled
 444                          * state.
 445                          */
 446                         new = 0;
 447                 }
 448         } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
 449
 450         return err;
 451 }
 452
 453 static int hw_perf_event_update(struct perf_event *event)
 454 {
 455         u64 prev, new, delta;
 456         int err;
 457
 458         do {
 459                 prev = local64_read(&event->hw.prev_count);
 460                 err = ecctr(event->hw.config, &new);
 461                 if (err)
 462                         goto out;
 463         } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
 464
 465         delta = (prev <= new) ? new - prev
 466                               : (-1ULL - prev) + new + 1;        /* overflow */
 467         local64_add(delta, &event->count);
 468 out:
 469         return err;
 470 }
 471
 472 static void cpumf_pmu_read(struct perf_event *event)
 473 {
 474         if (event->hw.state & PERF_HES_STOPPED)
 475                 return;
 476
 477         hw_perf_event_update(event);
 478 }
 479
 480 static void cpumf_pmu_start(struct perf_event *event, int flags)
 481 {
 482         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 483         struct hw_perf_event *hwc = &event->hw;
 484
 485         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 486                 return;
 487
 488         if (WARN_ON_ONCE(hwc->config == -1))
 489                 return;
 490
 491         if (flags & PERF_EF_RELOAD)
 492                 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 493
 494         hwc->state = 0;
 495
 496         /* (Re-)enable and activate the counter set */
 497         ctr_set_enable(&cpuhw->state, hwc->config_base);
 498         ctr_set_start(&cpuhw->state, hwc->config_base);
 499
 500         /* The counter set to which this counter belongs can be already active.
 501          * Because all counters in a set are active, the event->hw.prev_count
 502          * needs to be synchronized.  At this point, the counter set can be in
 503          * the inactive or disabled state.
 504          */
 505         hw_perf_event_reset(event);
 506
 507         /* increment refcount for this counter set */
 508         atomic_inc(&cpuhw->ctr_set[hwc->config_base]);
 509 }
 510
 511 static void cpumf_pmu_stop(struct perf_event *event, int flags)
 512 {
 513         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 514         struct hw_perf_event *hwc = &event->hw;
 515
 516         if (!(hwc->state & PERF_HES_STOPPED)) {
 517                 /* Decrement reference count for this counter set and if this
 518                  * is the last used counter in the set, clear activation
 519                  * control and set the counter set state to inactive.
 520                  */
 521                 if (!atomic_dec_return(&cpuhw->ctr_set[hwc->config_base]))
 522                         ctr_set_stop(&cpuhw->state, hwc->config_base);
 523                 event->hw.state |= PERF_HES_STOPPED;
 524         }
 525
 526         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
 527                 hw_perf_event_update(event);
 528                 event->hw.state |= PERF_HES_UPTODATE;
 529         }
 530 }
 531
 532 static int cpumf_pmu_add(struct perf_event *event, int flags)
 533 {
 534         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 535
 536         /* Check authorization for the counter set to which this
 537          * counter belongs.
 538          * For group events transaction, the authorization check is
 539          * done in cpumf_pmu_commit_txn().
 540          */
 541         if (!(cpuhw->flags & PERF_EVENT_TXN))
 542                 if (validate_ctr_auth(&event->hw))
 543                         return -ENOENT;
 544
 545         ctr_set_enable(&cpuhw->state, event->hw.config_base);
 546         event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 547
 548         if (flags & PERF_EF_START)
 549                 cpumf_pmu_start(event, PERF_EF_RELOAD);
 550
 551         perf_event_update_userpage(event);
 552
 553         return 0;
 554 }
 555
 556 static void cpumf_pmu_del(struct perf_event *event, int flags)
 557 {
 558         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 559
 560         cpumf_pmu_stop(event, PERF_EF_UPDATE);
 561
 562         /* Check if any counter in the counter set is still used.  If not used,
 563          * change the counter set to the disabled state.  This also clears the
 564          * content of all counters in the set.
 565          *
 566          * When a new perf event has been added but not yet started, this can
 567          * clear enable control and resets all counters in a set.  Therefore,
 568          * cpumf_pmu_start() always has to reenable a counter set.
 569          */
 570         if (!atomic_read(&cpuhw->ctr_set[event->hw.config_base]))
 571                 ctr_set_disable(&cpuhw->state, event->hw.config_base);
 572
 573         perf_event_update_userpage(event);
 574 }
 575
 576 /*
 577  * Start group events scheduling transaction.
 578  * Set flags to perform a single test at commit time.
 579  */
 580 static void cpumf_pmu_start_txn(struct pmu *pmu)
 581 {
 582         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 583
 584         perf_pmu_disable(pmu);
 585         cpuhw->flags |= PERF_EVENT_TXN;
 586         cpuhw->tx_state = cpuhw->state;
 587 }
 588
 589 /*
 590  * Stop and cancel a group events scheduling tranctions.
 591  * Assumes cpumf_pmu_del() is called for each successful added
 592  * cpumf_pmu_add() during the transaction.
 593  */
 594 static void cpumf_pmu_cancel_txn(struct pmu *pmu)
 595 {
 596         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 597
 598         WARN_ON(cpuhw->tx_state != cpuhw->state);
 599
 600         cpuhw->flags &= ~PERF_EVENT_TXN;
 601         perf_pmu_enable(pmu);
 602 }
 603
 604 /*
 605  * Commit the group events scheduling transaction.  On success, the
 606  * transaction is closed.   On error, the transaction is kept open
 607  * until cpumf_pmu_cancel_txn() is called.
 608  */
 609 static int cpumf_pmu_commit_txn(struct pmu *pmu)
 610 {
 611         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 612         u64 state;
 613
 614         /* check if the updated state can be scheduled */
 615         state = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
 616         state >>= CPUMF_LCCTL_ENABLE_SHIFT;
 617         if ((state & cpuhw->info.auth_ctl) != state)
 618                 return -ENOENT;
 619
 620         cpuhw->flags &= ~PERF_EVENT_TXN;
 621         perf_pmu_enable(pmu);
 622         return 0;
 623 }
 624
 625 /* Performance monitoring unit for s390x */
 626 static struct pmu cpumf_pmu = {
 627         .pmu_enable   = cpumf_pmu_enable,
 628         .pmu_disable  = cpumf_pmu_disable,
 629         .event_init   = cpumf_pmu_event_init,
 630         .add          = cpumf_pmu_add,
 631         .del          = cpumf_pmu_del,
 632         .start        = cpumf_pmu_start,
 633         .stop         = cpumf_pmu_stop,
 634         .read         = cpumf_pmu_read,
 635         .start_txn    = cpumf_pmu_start_txn,
 636         .commit_txn   = cpumf_pmu_commit_txn,
 637         .cancel_txn   = cpumf_pmu_cancel_txn,
 638 };
 639
 640 static int cpumf_pmu_notifier(struct notifier_block *self, unsigned long action,
 641                               void *hcpu)
 642 {
 643         unsigned int cpu = (long) hcpu;
 644         int flags;
 645
 646         switch (action & ~CPU_TASKS_FROZEN) {
 647         case CPU_ONLINE:
 648                 flags = PMC_INIT;
 649                 smp_call_function_single(cpu, setup_pmc_cpu, &flags, 1);
 650                 break;
 651         case CPU_DOWN_PREPARE:
 652                 flags = PMC_RELEASE;
 653                 smp_call_function_single(cpu, setup_pmc_cpu, &flags, 1);
 654                 break;
 655         default:
 656                 break;
 657         }
 658
 659         return NOTIFY_OK;
 660 }
 661
 662 static int __init cpumf_pmu_init(void)
 663 {
 664         int rc;
 665
 666         if (!cpum_cf_avail())
 667                 return -ENODEV;
 668
 669         /* clear bit 15 of cr0 to unauthorize problem-state to
 670          * extract measurement counters */
 671         ctl_clear_bit(0, 48);
 672
 673         /* register handler for measurement-alert interruptions */
 674         rc = register_external_irq(EXT_IRQ_MEASURE_ALERT,
 675                                    cpumf_measurement_alert);
 676         if (rc) {
 677                 pr_err("Registering for CPU-measurement alerts "
 678                        "failed with rc=%i\n", rc);
 679                 goto out;
 680         }
 681
 682         /* The CPU measurement counter facility does not have overflow
 683          * interrupts to do sampling.  Sampling must be provided by
 684          * external means, for example, by timers.
 685          */
 686         cpumf_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
 687
 688         cpumf_pmu.attr_groups = cpumf_cf_event_group();
 689         rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW);
 690         if (rc) {
 691                 pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc);
 692                 unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
 693                                         cpumf_measurement_alert);
 694                 goto out;
 695         }
 696         perf_cpu_notifier(cpumf_pmu_notifier);
 697 out:
 698         return rc;
 699 }
 700 early_initcall(cpumf_pmu_init);