lib/percpu_counter.c

   1 /*
   2  * Fast batching percpu counters.
   3  */
   4
   5 #include <linux/percpu_counter.h>
   6 #include <linux/module.h>
   7
   8 void percpu_counter_mod(struct percpu_counter *fbc, s32 amount)
   9 {
  10         long count;
  11         s32 *pcount;
  12         int cpu = get_cpu();
  13
  14         pcount = per_cpu_ptr(fbc->counters, cpu);
  15         count = *pcount + amount;
  16         if (count >= FBC_BATCH || count <= -FBC_BATCH) {
  17                 spin_lock(&fbc->lock);
  18                 fbc->count += count;
  19                 *pcount = 0;
  20                 spin_unlock(&fbc->lock);
  21         } else {
  22                 *pcount = count;
  23         }
  24         put_cpu();
  25 }
  26 EXPORT_SYMBOL(percpu_counter_mod);
  27
  28 /*
  29  * Add up all the per-cpu counts, return the result.  This is a more accurate
  30  * but much slower version of percpu_counter_read_positive()
  31  */
  32 s64 percpu_counter_sum(struct percpu_counter *fbc)
  33 {
  34         s64 ret;
  35         int cpu;
  36
  37         spin_lock(&fbc->lock);
  38         ret = fbc->count;
  39         for_each_possible_cpu(cpu) {
  40                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
  41                 ret += *pcount;
  42         }
  43         spin_unlock(&fbc->lock);
  44         return ret < 0 ? 0 : ret;
  45 }
  46 EXPORT_SYMBOL(percpu_counter_sum);