arch/x86/kernel/cpu/perf_event_intel_bts.c

   1 /*
   2  * BTS PMU driver for perf
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #undef DEBUG
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/bitops.h>
  20 #include <linux/types.h>
  21 #include <linux/slab.h>
  22 #include <linux/debugfs.h>
  23 #include <linux/device.h>
  24 #include <linux/coredump.h>
  25
  26 #include <asm-generic/sizes.h>
  27 #include <asm/perf_event.h>
  28
  29 #include "perf_event.h"
  30
  31 struct bts_ctx {
  32         struct perf_output_handle       handle;
  33         struct debug_store              ds_back;
  34         int                             started;
  35 };
  36
  37 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
  38
  39 #define BTS_RECORD_SIZE         24
  40 #define BTS_SAFETY_MARGIN       4080
  41
  42 struct bts_phys {
  43         struct page     *page;
  44         unsigned long   size;
  45         unsigned long   offset;
  46         unsigned long   displacement;
  47 };
  48
  49 struct bts_buffer {
  50         size_t          real_size;      /* multiple of BTS_RECORD_SIZE */
  51         unsigned int    nr_pages;
  52         unsigned int    nr_bufs;
  53         unsigned int    cur_buf;
  54         bool            snapshot;
  55         local_t         data_size;
  56         local_t         lost;
  57         local_t         head;
  58         unsigned long   end;
  59         void            **data_pages;
  60         struct bts_phys buf[0];
  61 };
  62
  63 struct pmu bts_pmu;
  64
  65 static size_t buf_size(struct page *page)
  66 {
  67         return 1 << (PAGE_SHIFT + page_private(page));
  68 }
  69
  70 static void *
  71 bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
  72 {
  73         struct bts_buffer *buf;
  74         struct page *page;
  75         int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  76         unsigned long offset;
  77         size_t size = nr_pages << PAGE_SHIFT;
  78         int pg, nbuf, pad;
  79
  80         /* count all the high order buffers */
  81         for (pg = 0, nbuf = 0; pg < nr_pages;) {
  82                 page = virt_to_page(pages[pg]);
  83                 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
  84                         return NULL;
  85                 pg += 1 << page_private(page);
  86                 nbuf++;
  87         }
  88
  89         /*
  90          * to avoid interrupts in overwrite mode, only allow one physical
  91          */
  92         if (overwrite && nbuf > 1)
  93                 return NULL;
  94
  95         buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
  96         if (!buf)
  97                 return NULL;
  98
  99         buf->nr_pages = nr_pages;
 100         buf->nr_bufs = nbuf;
 101         buf->snapshot = overwrite;
 102         buf->data_pages = pages;
 103         buf->real_size = size - size % BTS_RECORD_SIZE;
 104
 105         for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
 106                 unsigned int __nr_pages;
 107
 108                 page = virt_to_page(pages[pg]);
 109                 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
 110                 buf->buf[nbuf].page = page;
 111                 buf->buf[nbuf].offset = offset;
 112                 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
 113                 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
 114                 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
 115                 buf->buf[nbuf].size -= pad;
 116
 117                 pg += __nr_pages;
 118                 offset += __nr_pages << PAGE_SHIFT;
 119         }
 120
 121         return buf;
 122 }
 123
 124 static void bts_buffer_free_aux(void *data)
 125 {
 126         kfree(data);
 127 }
 128
 129 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
 130 {
 131         return buf->buf[idx].offset + buf->buf[idx].displacement;
 132 }
 133
 134 static void
 135 bts_config_buffer(struct bts_buffer *buf)
 136 {
 137         int cpu = raw_smp_processor_id();
 138         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 139         struct bts_phys *phys = &buf->buf[buf->cur_buf];
 140         unsigned long index, thresh = 0, end = phys->size;
 141         struct page *page = phys->page;
 142
 143         index = local_read(&buf->head);
 144
 145         if (!buf->snapshot) {
 146                 if (buf->end < phys->offset + buf_size(page))
 147                         end = buf->end - phys->offset - phys->displacement;
 148
 149                 index -= phys->offset + phys->displacement;
 150
 151                 if (end - index > BTS_SAFETY_MARGIN)
 152                         thresh = end - BTS_SAFETY_MARGIN;
 153                 else if (end - index > BTS_RECORD_SIZE)
 154                         thresh = end - BTS_RECORD_SIZE;
 155                 else
 156                         thresh = end;
 157         }
 158
 159         ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 160         ds->bts_index = ds->bts_buffer_base + index;
 161         ds->bts_absolute_maximum = ds->bts_buffer_base + end;
 162         ds->bts_interrupt_threshold = !buf->snapshot
 163                 ? ds->bts_buffer_base + thresh
 164                 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 165 }
 166
 167 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
 168 {
 169         unsigned long index = head - phys->offset;
 170
 171         memset(page_address(phys->page) + index, 0, phys->size - index);
 172 }
 173
 174 static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)
 175 {
 176         if (buf->snapshot)
 177                 return false;
 178
 179         if (local_read(&buf->data_size) >= bts->handle.size ||
 180             bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)
 181                 return true;
 182
 183         return false;
 184 }
 185
 186 static void bts_update(struct bts_ctx *bts)
 187 {
 188         int cpu = raw_smp_processor_id();
 189         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 190         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 191         unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
 192
 193         if (!buf)
 194                 return;
 195
 196         head = index + bts_buffer_offset(buf, buf->cur_buf);
 197         old = local_xchg(&buf->head, head);
 198
 199         if (!buf->snapshot) {
 200                 if (old == head)
 201                         return;
 202
 203                 if (ds->bts_index >= ds->bts_absolute_maximum)
 204                         local_inc(&buf->lost);
 205
 206                 /*
 207                  * old and head are always in the same physical buffer, so we
 208                  * can subtract them to get the data size.
 209                  */
 210                 local_add(head - old, &buf->data_size);
 211         } else {
 212                 local_set(&buf->data_size, head);
 213         }
 214 }
 215
 216 static void __bts_event_start(struct perf_event *event)
 217 {
 218         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 219         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 220         u64 config = 0;
 221
 222         if (!buf || bts_buffer_is_full(buf, bts))
 223                 return;
 224
 225         event->hw.itrace_started = 1;
 226         event->hw.state = 0;
 227
 228         if (!buf->snapshot)
 229                 config |= ARCH_PERFMON_EVENTSEL_INT;
 230         if (!event->attr.exclude_kernel)
 231                 config |= ARCH_PERFMON_EVENTSEL_OS;
 232         if (!event->attr.exclude_user)
 233                 config |= ARCH_PERFMON_EVENTSEL_USR;
 234
 235         bts_config_buffer(buf);
 236
 237         /*
 238          * local barrier to make sure that ds configuration made it
 239          * before we enable BTS
 240          */
 241         wmb();
 242
 243         intel_pmu_enable_bts(config);
 244 }
 245
 246 static void bts_event_start(struct perf_event *event, int flags)
 247 {
 248         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 249
 250         __bts_event_start(event);
 251
 252         /* PMI handler: this counter is running and likely generating PMIs */
 253         ACCESS_ONCE(bts->started) = 1;
 254 }
 255
 256 static void __bts_event_stop(struct perf_event *event)
 257 {
 258         /*
 259          * No extra synchronization is mandated by the documentation to have
 260          * BTS data stores globally visible.
 261          */
 262         intel_pmu_disable_bts();
 263
 264         if (event->hw.state & PERF_HES_STOPPED)
 265                 return;
 266
 267         ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
 268 }
 269
 270 static void bts_event_stop(struct perf_event *event, int flags)
 271 {
 272         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 273
 274         /* PMI handler: don't restart this counter */
 275         ACCESS_ONCE(bts->started) = 0;
 276
 277         __bts_event_stop(event);
 278
 279         if (flags & PERF_EF_UPDATE)
 280                 bts_update(bts);
 281 }
 282
 283 void intel_bts_enable_local(void)
 284 {
 285         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 286
 287         if (bts->handle.event && bts->started)
 288                 __bts_event_start(bts->handle.event);
 289 }
 290
 291 void intel_bts_disable_local(void)
 292 {
 293         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 294
 295         if (bts->handle.event)
 296                 __bts_event_stop(bts->handle.event);
 297 }
 298
 299 static int
 300 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 301 {
 302         unsigned long head, space, next_space, pad, gap, skip, wakeup;
 303         unsigned int next_buf;
 304         struct bts_phys *phys, *next_phys;
 305         int ret;
 306
 307         if (buf->snapshot)
 308                 return 0;
 309
 310         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 311         if (WARN_ON_ONCE(head != local_read(&buf->head)))
 312                 return -EINVAL;
 313
 314         phys = &buf->buf[buf->cur_buf];
 315         space = phys->offset + phys->displacement + phys->size - head;
 316         pad = space;
 317         if (space > handle->size) {
 318                 space = handle->size;
 319                 space -= space % BTS_RECORD_SIZE;
 320         }
 321         if (space <= BTS_SAFETY_MARGIN) {
 322                 /* See if next phys buffer has more space */
 323                 next_buf = buf->cur_buf + 1;
 324                 if (next_buf >= buf->nr_bufs)
 325                         next_buf = 0;
 326                 next_phys = &buf->buf[next_buf];
 327                 gap = buf_size(phys->page) - phys->displacement - phys->size +
 328                       next_phys->displacement;
 329                 skip = pad + gap;
 330                 if (handle->size >= skip) {
 331                         next_space = next_phys->size;
 332                         if (next_space + skip > handle->size) {
 333                                 next_space = handle->size - skip;
 334                                 next_space -= next_space % BTS_RECORD_SIZE;
 335                         }
 336                         if (next_space > space || !space) {
 337                                 if (pad)
 338                                         bts_buffer_pad_out(phys, head);
 339                                 ret = perf_aux_output_skip(handle, skip);
 340                                 if (ret)
 341                                         return ret;
 342                                 /* Advance to next phys buffer */
 343                                 phys = next_phys;
 344                                 space = next_space;
 345                                 head = phys->offset + phys->displacement;
 346                                 /*
 347                                  * After this, cur_buf and head won't match ds
 348                                  * anymore, so we must not be racing with
 349                                  * bts_update().
 350                                  */
 351                                 buf->cur_buf = next_buf;
 352                                 local_set(&buf->head, head);
 353                         }
 354                 }
 355         }
 356
 357         /* Don't go far beyond wakeup watermark */
 358         wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
 359                  handle->head;
 360         if (space > wakeup) {
 361                 space = wakeup;
 362                 space -= space % BTS_RECORD_SIZE;
 363         }
 364
 365         buf->end = head + space;
 366
 367         /*
 368          * If we have no space, the lost notification would have been sent when
 369          * we hit absolute_maximum - see bts_update()
 370          */
 371         if (!space)
 372                 return -ENOSPC;
 373
 374         return 0;
 375 }
 376
 377 int intel_bts_interrupt(void)
 378 {
 379         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 380         struct perf_event *event = bts->handle.event;
 381         struct bts_buffer *buf;
 382         s64 old_head;
 383         int err;
 384
 385         if (!event || !bts->started)
 386                 return 0;
 387
 388         buf = perf_get_aux(&bts->handle);
 389         /*
 390          * Skip snapshot counters: they don't use the interrupt, but
 391          * there's no other way of telling, because the pointer will
 392          * keep moving
 393          */
 394         if (!buf || buf->snapshot)
 395                 return 0;
 396
 397         old_head = local_read(&buf->head);
 398         bts_update(bts);
 399
 400         /* no new data */
 401         if (old_head == local_read(&buf->head))
 402                 return 0;
 403
 404         perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
 405                             !!local_xchg(&buf->lost, 0));
 406
 407         buf = perf_aux_output_begin(&bts->handle, event);
 408         if (!buf)
 409                 return 1;
 410
 411         err = bts_buffer_reset(buf, &bts->handle);
 412         if (err)
 413                 perf_aux_output_end(&bts->handle, 0, false);
 414
 415         return 1;
 416 }
 417
 418 static void bts_event_del(struct perf_event *event, int mode)
 419 {
 420         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 421         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 422         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 423
 424         bts_event_stop(event, PERF_EF_UPDATE);
 425
 426         if (buf) {
 427                 if (buf->snapshot)
 428                         bts->handle.head =
 429                                 local_xchg(&buf->data_size,
 430                                            buf->nr_pages << PAGE_SHIFT);
 431                 perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
 432                                     !!local_xchg(&buf->lost, 0));
 433         }
 434
 435         cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
 436         cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
 437         cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
 438         cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
 439 }
 440
 441 static int bts_event_add(struct perf_event *event, int mode)
 442 {
 443         struct bts_buffer *buf;
 444         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 445         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 446         struct hw_perf_event *hwc = &event->hw;
 447         int ret = -EBUSY;
 448
 449         event->hw.state = PERF_HES_STOPPED;
 450
 451         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 452                 return -EBUSY;
 453
 454         if (bts->handle.event)
 455                 return -EBUSY;
 456
 457         buf = perf_aux_output_begin(&bts->handle, event);
 458         if (!buf)
 459                 return -EINVAL;
 460
 461         ret = bts_buffer_reset(buf, &bts->handle);
 462         if (ret) {
 463                 perf_aux_output_end(&bts->handle, 0, false);
 464                 return ret;
 465         }
 466
 467         bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
 468         bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
 469         bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
 470
 471         if (mode & PERF_EF_START) {
 472                 bts_event_start(event, 0);
 473                 if (hwc->state & PERF_HES_STOPPED) {
 474                         bts_event_del(event, 0);
 475                         return -EBUSY;
 476                 }
 477         }
 478
 479         return 0;
 480 }
 481
 482 static void bts_event_destroy(struct perf_event *event)
 483 {
 484         x86_release_hardware();
 485         x86_del_exclusive(x86_lbr_exclusive_bts);
 486 }
 487
 488 static int bts_event_init(struct perf_event *event)
 489 {
 490         int ret;
 491
 492         if (event->attr.type != bts_pmu.type)
 493                 return -ENOENT;
 494
 495         if (x86_add_exclusive(x86_lbr_exclusive_bts))
 496                 return -EBUSY;
 497
 498         ret = x86_reserve_hardware();
 499         if (ret) {
 500                 x86_del_exclusive(x86_lbr_exclusive_bts);
 501                 return ret;
 502         }
 503
 504         event->destroy = bts_event_destroy;
 505
 506         return 0;
 507 }
 508
 509 static void bts_event_read(struct perf_event *event)
 510 {
 511 }
 512
 513 static __init int bts_init(void)
 514 {
 515         if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
 516                 return -ENODEV;
 517
 518         bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE;
 519         bts_pmu.task_ctx_nr     = perf_sw_context;
 520         bts_pmu.event_init      = bts_event_init;
 521         bts_pmu.add             = bts_event_add;
 522         bts_pmu.del             = bts_event_del;
 523         bts_pmu.start           = bts_event_start;
 524         bts_pmu.stop            = bts_event_stop;
 525         bts_pmu.read            = bts_event_read;
 526         bts_pmu.setup_aux       = bts_buffer_setup_aux;
 527         bts_pmu.free_aux        = bts_buffer_free_aux;
 528
 529         return perf_pmu_register(&bts_pmu, "intel_bts", -1);
 530 }
 531 arch_initcall(bts_init);