kernel/tracepoint.c

   1 /*
   2  * Copyright (C) 2008 Mathieu Desnoyers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  */
  18 #include <linux/module.h>
  19 #include <linux/mutex.h>
  20 #include <linux/types.h>
  21 #include <linux/jhash.h>
  22 #include <linux/list.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/tracepoint.h>
  25 #include <linux/err.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched.h>
  28 #include <linux/static_key.h>
  29
  30 extern struct tracepoint * const __start___tracepoints_ptrs[];
  31 extern struct tracepoint * const __stop___tracepoints_ptrs[];
  32
  33 /* Set to 1 to enable tracepoint debug output */
  34 static const int tracepoint_debug;
  35
  36 /*
  37  * Tracepoints mutex protects the builtin and module tracepoints and the hash
  38  * table, as well as the local module list.
  39  */
  40 static DEFINE_MUTEX(tracepoints_mutex);
  41
  42 #ifdef CONFIG_MODULES
  43 /* Local list of struct module */
  44 static LIST_HEAD(tracepoint_module_list);
  45 #endif /* CONFIG_MODULES */
  46
  47 /*
  48  * Tracepoint hash table, containing the active tracepoints.
  49  * Protected by tracepoints_mutex.
  50  */
  51 #define TRACEPOINT_HASH_BITS 6
  52 #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
  53 static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
  54
  55 /*
  56  * Note about RCU :
  57  * It is used to delay the free of multiple probes array until a quiescent
  58  * state is reached.
  59  * Tracepoint entries modifications are protected by the tracepoints_mutex.
  60  */
  61 struct tracepoint_entry {
  62         struct hlist_node hlist;
  63         struct tracepoint_func *funcs;
  64         int refcount;   /* Number of times armed. 0 if disarmed. */
  65         int enabled;    /* Tracepoint enabled */
  66         char name[0];
  67 };
  68
  69 struct tp_probes {
  70         struct rcu_head rcu;
  71         struct tracepoint_func probes[0];
  72 };
  73
  74 static inline void *allocate_probes(int count)
  75 {
  76         struct tp_probes *p  = kmalloc(count * sizeof(struct tracepoint_func)
  77                         + sizeof(struct tp_probes), GFP_KERNEL);
  78         return p == NULL ? NULL : p->probes;
  79 }
  80
  81 static void rcu_free_old_probes(struct rcu_head *head)
  82 {
  83         kfree(container_of(head, struct tp_probes, rcu));
  84 }
  85
  86 static inline void release_probes(struct tracepoint_func *old)
  87 {
  88         if (old) {
  89                 struct tp_probes *tp_probes = container_of(old,
  90                         struct tp_probes, probes[0]);
  91                 call_rcu_sched(&tp_probes->rcu, rcu_free_old_probes);
  92         }
  93 }
  94
  95 static void debug_print_probes(struct tracepoint_entry *entry)
  96 {
  97         int i;
  98
  99         if (!tracepoint_debug || !entry->funcs)
 100                 return;
 101
 102         for (i = 0; entry->funcs[i].func; i++)
 103                 printk(KERN_DEBUG "Probe %d : %p\n", i, entry->funcs[i].func);
 104 }
 105
 106 static struct tracepoint_func *
 107 tracepoint_entry_add_probe(struct tracepoint_entry *entry,
 108                            void *probe, void *data)
 109 {
 110         int nr_probes = 0;
 111         struct tracepoint_func *old, *new;
 112
 113         if (WARN_ON(!probe))
 114                 return ERR_PTR(-EINVAL);
 115
 116         debug_print_probes(entry);
 117         old = entry->funcs;
 118         if (old) {
 119                 /* (N -> N+1), (N != 0, 1) probes */
 120                 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
 121                         if (old[nr_probes].func == probe &&
 122                             old[nr_probes].data == data)
 123                                 return ERR_PTR(-EEXIST);
 124         }
 125         /* + 2 : one for new probe, one for NULL func */
 126         new = allocate_probes(nr_probes + 2);
 127         if (new == NULL)
 128                 return ERR_PTR(-ENOMEM);
 129         if (old)
 130                 memcpy(new, old, nr_probes * sizeof(struct tracepoint_func));
 131         new[nr_probes].func = probe;
 132         new[nr_probes].data = data;
 133         new[nr_probes + 1].func = NULL;
 134         entry->refcount = nr_probes + 1;
 135         entry->funcs = new;
 136         debug_print_probes(entry);
 137         return old;
 138 }
 139
 140 static void *
 141 tracepoint_entry_remove_probe(struct tracepoint_entry *entry,
 142                               void *probe, void *data)
 143 {
 144         int nr_probes = 0, nr_del = 0, i;
 145         struct tracepoint_func *old, *new;
 146
 147         old = entry->funcs;
 148
 149         if (!old)
 150                 return ERR_PTR(-ENOENT);
 151
 152         debug_print_probes(entry);
 153         /* (N -> M), (N > 1, M >= 0) probes */
 154         if (probe) {
 155                 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 156                         if (old[nr_probes].func == probe &&
 157                              old[nr_probes].data == data)
 158                                 nr_del++;
 159                 }
 160         }
 161
 162         /*
 163          * If probe is NULL, then nr_probes = nr_del = 0, and then the
 164          * entire entry will be removed.
 165          */
 166         if (nr_probes - nr_del == 0) {
 167                 /* N -> 0, (N > 1) */
 168                 entry->funcs = NULL;
 169                 entry->refcount = 0;
 170                 debug_print_probes(entry);
 171                 return old;
 172         } else {
 173                 int j = 0;
 174                 /* N -> M, (N > 1, M > 0) */
 175                 /* + 1 for NULL */
 176                 new = allocate_probes(nr_probes - nr_del + 1);
 177                 if (new == NULL)
 178                         return ERR_PTR(-ENOMEM);
 179                 for (i = 0; old[i].func; i++)
 180                         if (old[i].func != probe || old[i].data != data)
 181                                 new[j++] = old[i];
 182                 new[nr_probes - nr_del].func = NULL;
 183                 entry->refcount = nr_probes - nr_del;
 184                 entry->funcs = new;
 185         }
 186         debug_print_probes(entry);
 187         return old;
 188 }
 189
 190 /*
 191  * Get tracepoint if the tracepoint is present in the tracepoint hash table.
 192  * Must be called with tracepoints_mutex held.
 193  * Returns NULL if not present.
 194  */
 195 static struct tracepoint_entry *get_tracepoint(const char *name)
 196 {
 197         struct hlist_head *head;
 198         struct tracepoint_entry *e;
 199         u32 hash = jhash(name, strlen(name), 0);
 200
 201         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 202         hlist_for_each_entry(e, head, hlist) {
 203                 if (!strcmp(name, e->name))
 204                         return e;
 205         }
 206         return NULL;
 207 }
 208
 209 /*
 210  * Add the tracepoint to the tracepoint hash table. Must be called with
 211  * tracepoints_mutex held.
 212  */
 213 static struct tracepoint_entry *add_tracepoint(const char *name)
 214 {
 215         struct hlist_head *head;
 216         struct tracepoint_entry *e;
 217         size_t name_len = strlen(name) + 1;
 218         u32 hash = jhash(name, name_len-1, 0);
 219
 220         head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
 221         hlist_for_each_entry(e, head, hlist) {
 222                 if (!strcmp(name, e->name)) {
 223                         printk(KERN_NOTICE
 224                                 "tracepoint %s busy\n", name);
 225                         return ERR_PTR(-EEXIST);        /* Already there */
 226                 }
 227         }
 228         /*
 229          * Using kmalloc here to allocate a variable length element. Could
 230          * cause some memory fragmentation if overused.
 231          */
 232         e = kmalloc(sizeof(struct tracepoint_entry) + name_len, GFP_KERNEL);
 233         if (!e)
 234                 return ERR_PTR(-ENOMEM);
 235         memcpy(&e->name[0], name, name_len);
 236         e->funcs = NULL;
 237         e->refcount = 0;
 238         e->enabled = 0;
 239         hlist_add_head(&e->hlist, head);
 240         return e;
 241 }
 242
 243 /*
 244  * Remove the tracepoint from the tracepoint hash table. Must be called with
 245  * mutex_lock held.
 246  */
 247 static inline void remove_tracepoint(struct tracepoint_entry *e)
 248 {
 249         hlist_del(&e->hlist);
 250         kfree(e);
 251 }
 252
 253 /*
 254  * Sets the probe callback corresponding to one tracepoint.
 255  */
 256 static void set_tracepoint(struct tracepoint_entry **entry,
 257         struct tracepoint *elem, int active)
 258 {
 259         WARN_ON(strcmp((*entry)->name, elem->name) != 0);
 260
 261         if (elem->regfunc && !static_key_enabled(&elem->key) && active)
 262                 elem->regfunc();
 263         else if (elem->unregfunc && static_key_enabled(&elem->key) && !active)
 264                 elem->unregfunc();
 265
 266         /*
 267          * rcu_assign_pointer has a smp_wmb() which makes sure that the new
 268          * probe callbacks array is consistent before setting a pointer to it.
 269          * This array is referenced by __DO_TRACE from
 270          * include/linux/tracepoints.h. A matching smp_read_barrier_depends()
 271          * is used.
 272          */
 273         rcu_assign_pointer(elem->funcs, (*entry)->funcs);
 274         if (active && !static_key_enabled(&elem->key))
 275                 static_key_slow_inc(&elem->key);
 276         else if (!active && static_key_enabled(&elem->key))
 277                 static_key_slow_dec(&elem->key);
 278 }
 279
 280 /*
 281  * Disable a tracepoint and its probe callback.
 282  * Note: only waiting an RCU period after setting elem->call to the empty
 283  * function insures that the original callback is not used anymore. This insured
 284  * by preempt_disable around the call site.
 285  */
 286 static void disable_tracepoint(struct tracepoint *elem)
 287 {
 288         if (elem->unregfunc && static_key_enabled(&elem->key))
 289                 elem->unregfunc();
 290
 291         if (static_key_enabled(&elem->key))
 292                 static_key_slow_dec(&elem->key);
 293         rcu_assign_pointer(elem->funcs, NULL);
 294 }
 295
 296 /**
 297  * tracepoint_update_probe_range - Update a probe range
 298  * @begin: beginning of the range
 299  * @end: end of the range
 300  *
 301  * Updates the probe callback corresponding to a range of tracepoints.
 302  * Called with tracepoints_mutex held.
 303  */
 304 static void tracepoint_update_probe_range(struct tracepoint * const *begin,
 305                                           struct tracepoint * const *end)
 306 {
 307         struct tracepoint * const *iter;
 308         struct tracepoint_entry *mark_entry;
 309
 310         if (!begin)
 311                 return;
 312
 313         for (iter = begin; iter < end; iter++) {
 314                 mark_entry = get_tracepoint((*iter)->name);
 315                 if (mark_entry) {
 316                         set_tracepoint(&mark_entry, *iter,
 317                                         !!mark_entry->refcount);
 318                         mark_entry->enabled = !!mark_entry->refcount;
 319                 } else {
 320                         disable_tracepoint(*iter);
 321                 }
 322         }
 323 }
 324
 325 #ifdef CONFIG_MODULES
 326 void module_update_tracepoints(void)
 327 {
 328         struct tp_module *tp_mod;
 329
 330         list_for_each_entry(tp_mod, &tracepoint_module_list, list)
 331                 tracepoint_update_probe_range(tp_mod->tracepoints_ptrs,
 332                         tp_mod->tracepoints_ptrs + tp_mod->num_tracepoints);
 333 }
 334 #else /* CONFIG_MODULES */
 335 void module_update_tracepoints(void)
 336 {
 337 }
 338 #endif /* CONFIG_MODULES */
 339
 340
 341 /*
 342  * Update probes, removing the faulty probes.
 343  * Called with tracepoints_mutex held.
 344  */
 345 static void tracepoint_update_probes(void)
 346 {
 347         /* Core kernel tracepoints */
 348         tracepoint_update_probe_range(__start___tracepoints_ptrs,
 349                 __stop___tracepoints_ptrs);
 350         /* tracepoints in modules. */
 351         module_update_tracepoints();
 352 }
 353
 354 static struct tracepoint_func *
 355 tracepoint_add_probe(const char *name, void *probe, void *data)
 356 {
 357         struct tracepoint_entry *entry;
 358         struct tracepoint_func *old;
 359
 360         entry = get_tracepoint(name);
 361         if (!entry) {
 362                 entry = add_tracepoint(name);
 363                 if (IS_ERR(entry))
 364                         return (struct tracepoint_func *)entry;
 365         }
 366         old = tracepoint_entry_add_probe(entry, probe, data);
 367         if (IS_ERR(old) && !entry->refcount)
 368                 remove_tracepoint(entry);
 369         return old;
 370 }
 371
 372 /**
 373  * tracepoint_probe_register -  Connect a probe to a tracepoint
 374  * @name: tracepoint name
 375  * @probe: probe handler
 376  * @data: probe private data
 377  *
 378  * Returns:
 379  * - 0 if the probe was successfully registered, and tracepoint
 380  *   callsites are currently loaded for that probe,
 381  * - -ENODEV if the probe was successfully registered, but no tracepoint
 382  *   callsite is currently loaded for that probe,
 383  * - other negative error value on error.
 384  *
 385  * When tracepoint_probe_register() returns either 0 or -ENODEV,
 386  * parameters @name, @probe, and @data may be used by the tracepoint
 387  * infrastructure until the probe is unregistered.
 388  *
 389  * The probe address must at least be aligned on the architecture pointer size.
 390  */
 391 int tracepoint_probe_register(const char *name, void *probe, void *data)
 392 {
 393         struct tracepoint_func *old;
 394         struct tracepoint_entry *entry;
 395         int ret = 0;
 396
 397         mutex_lock(&tracepoints_mutex);
 398         old = tracepoint_add_probe(name, probe, data);
 399         if (IS_ERR(old)) {
 400                 mutex_unlock(&tracepoints_mutex);
 401                 return PTR_ERR(old);
 402         }
 403         tracepoint_update_probes();             /* may update entry */
 404         entry = get_tracepoint(name);
 405         /* Make sure the entry was enabled */
 406         if (!entry || !entry->enabled)
 407                 ret = -ENODEV;
 408         mutex_unlock(&tracepoints_mutex);
 409         release_probes(old);
 410         return ret;
 411 }
 412 EXPORT_SYMBOL_GPL(tracepoint_probe_register);
 413
 414 static struct tracepoint_func *
 415 tracepoint_remove_probe(const char *name, void *probe, void *data)
 416 {
 417         struct tracepoint_entry *entry;
 418         struct tracepoint_func *old;
 419
 420         entry = get_tracepoint(name);
 421         if (!entry)
 422                 return ERR_PTR(-ENOENT);
 423         old = tracepoint_entry_remove_probe(entry, probe, data);
 424         if (IS_ERR(old))
 425                 return old;
 426         if (!entry->refcount)
 427                 remove_tracepoint(entry);
 428         return old;
 429 }
 430
 431 /**
 432  * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
 433  * @name: tracepoint name
 434  * @probe: probe function pointer
 435  * @data: probe private data
 436  *
 437  * We do not need to call a synchronize_sched to make sure the probes have
 438  * finished running before doing a module unload, because the module unload
 439  * itself uses stop_machine(), which insures that every preempt disabled section
 440  * have finished.
 441  */
 442 int tracepoint_probe_unregister(const char *name, void *probe, void *data)
 443 {
 444         struct tracepoint_func *old;
 445
 446         mutex_lock(&tracepoints_mutex);
 447         old = tracepoint_remove_probe(name, probe, data);
 448         if (IS_ERR(old)) {
 449                 mutex_unlock(&tracepoints_mutex);
 450                 return PTR_ERR(old);
 451         }
 452         tracepoint_update_probes();             /* may update entry */
 453         mutex_unlock(&tracepoints_mutex);
 454         release_probes(old);
 455         return 0;
 456 }
 457 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
 458
 459
 460 #ifdef CONFIG_MODULES
 461 bool trace_module_has_bad_taint(struct module *mod)
 462 {
 463         return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) |
 464                                (1 << TAINT_UNSIGNED_MODULE));
 465 }
 466
 467 static int tracepoint_module_coming(struct module *mod)
 468 {
 469         struct tp_module *tp_mod;
 470         int ret = 0;
 471
 472         if (!mod->num_tracepoints)
 473                 return 0;
 474
 475         /*
 476          * We skip modules that taint the kernel, especially those with different
 477          * module headers (for forced load), to make sure we don't cause a crash.
 478          * Staging, out-of-tree, and unsigned GPL modules are fine.
 479          */
 480         if (trace_module_has_bad_taint(mod))
 481                 return 0;
 482         mutex_lock(&tracepoints_mutex);
 483         tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
 484         if (!tp_mod) {
 485                 ret = -ENOMEM;
 486                 goto end;
 487         }
 488         tp_mod->num_tracepoints = mod->num_tracepoints;
 489         tp_mod->tracepoints_ptrs = mod->tracepoints_ptrs;
 490         list_add_tail(&tp_mod->list, &tracepoint_module_list);
 491         tracepoint_update_probe_range(mod->tracepoints_ptrs,
 492                 mod->tracepoints_ptrs + mod->num_tracepoints);
 493 end:
 494         mutex_unlock(&tracepoints_mutex);
 495         return ret;
 496 }
 497
 498 static int tracepoint_module_going(struct module *mod)
 499 {
 500         struct tp_module *pos;
 501
 502         if (!mod->num_tracepoints)
 503                 return 0;
 504
 505         mutex_lock(&tracepoints_mutex);
 506         tracepoint_update_probe_range(mod->tracepoints_ptrs,
 507                 mod->tracepoints_ptrs + mod->num_tracepoints);
 508         list_for_each_entry(pos, &tracepoint_module_list, list) {
 509                 if (pos->tracepoints_ptrs == mod->tracepoints_ptrs) {
 510                         list_del(&pos->list);
 511                         kfree(pos);
 512                         break;
 513                 }
 514         }
 515         /*
 516          * In the case of modules that were tainted at "coming", we'll simply
 517          * walk through the list without finding it. We cannot use the "tainted"
 518          * flag on "going", in case a module taints the kernel only after being
 519          * loaded.
 520          */
 521         mutex_unlock(&tracepoints_mutex);
 522         return 0;
 523 }
 524
 525 int tracepoint_module_notify(struct notifier_block *self,
 526                              unsigned long val, void *data)
 527 {
 528         struct module *mod = data;
 529         int ret = 0;
 530
 531         switch (val) {
 532         case MODULE_STATE_COMING:
 533                 ret = tracepoint_module_coming(mod);
 534                 break;
 535         case MODULE_STATE_LIVE:
 536                 break;
 537         case MODULE_STATE_GOING:
 538                 ret = tracepoint_module_going(mod);
 539                 break;
 540         }
 541         return ret;
 542 }
 543
 544 struct notifier_block tracepoint_module_nb = {
 545         .notifier_call = tracepoint_module_notify,
 546         .priority = 0,
 547 };
 548
 549 static int init_tracepoints(void)
 550 {
 551         return register_module_notifier(&tracepoint_module_nb);
 552 }
 553 __initcall(init_tracepoints);
 554 #endif /* CONFIG_MODULES */
 555
 556 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
 557
 558 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
 559 static int sys_tracepoint_refcount;
 560
 561 void syscall_regfunc(void)
 562 {
 563         unsigned long flags;
 564         struct task_struct *g, *t;
 565
 566         if (!sys_tracepoint_refcount) {
 567                 read_lock_irqsave(&tasklist_lock, flags);
 568                 do_each_thread(g, t) {
 569                         /* Skip kernel threads. */
 570                         if (t->mm)
 571                                 set_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
 572                 } while_each_thread(g, t);
 573                 read_unlock_irqrestore(&tasklist_lock, flags);
 574         }
 575         sys_tracepoint_refcount++;
 576 }
 577
 578 void syscall_unregfunc(void)
 579 {
 580         unsigned long flags;
 581         struct task_struct *g, *t;
 582
 583         sys_tracepoint_refcount--;
 584         if (!sys_tracepoint_refcount) {
 585                 read_lock_irqsave(&tasklist_lock, flags);
 586                 do_each_thread(g, t) {
 587                         clear_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
 588                 } while_each_thread(g, t);
 589                 read_unlock_irqrestore(&tasklist_lock, flags);
 590         }
 591 }
 592 #endif