arch/arc/include/asm/bitops.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef _ASM_BITOPS_H
  10 #define _ASM_BITOPS_H
  11
  12 #ifndef _LINUX_BITOPS_H
  13 #error only <linux/bitops.h> can be included directly
  14 #endif
  15
  16 #ifndef __ASSEMBLY__
  17
  18 #include <linux/types.h>
  19 #include <linux/compiler.h>
  20 #include <asm/barrier.h>
  21 #ifndef CONFIG_ARC_HAS_LLSC
  22 #include <asm/smp.h>
  23 #endif
  24
  25 #if defined(CONFIG_ARC_HAS_LLSC)
  26
  27 /*
  28  * Hardware assisted Atomic-R-M-W
  29  */
  30
  31 #define BIT_OP(op, c_op, asm_op)                                        \
  32 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
  33 {                                                                       \
  34         unsigned int temp;                                              \
  35                                                                         \
  36         m += nr >> 5;                                                   \
  37                                                                         \
  38         /*                                                              \
  39          * ARC ISA micro-optimization:                                  \
  40          *                                                              \
  41          * Instructions dealing with bitpos only consider lower 5 bits  \
  42          * e.g (x << 33) is handled like (x << 1) by ASL instruction    \
  43          *  (mem pointer still needs adjustment to point to next word)  \
  44          *                                                              \
  45          * Hence the masking to clamp @nr arg can be elided in general. \
  46          *                                                              \
  47          * However if @nr is a constant (above assumed in a register),  \
  48          * and greater than 31, gcc can optimize away (x << 33) to 0,   \
  49          * as overflow, given the 32-bit ISA. Thus masking needs to be  \
  50          * done for const @nr, but no code is generated due to gcc      \
  51          * const prop.                                                  \
  52          */                                                             \
  53         if (__builtin_constant_p(nr))                                   \
  54                 nr &= 0x1f;                                             \
  55                                                                         \
  56         __asm__ __volatile__(                                           \
  57         "1:     llock       %0, [%1]            \n"                     \
  58         "       " #asm_op " %0, %0, %2  \n"                             \
  59         "       scond       %0, [%1]            \n"                     \
  60         "       bnz         1b                  \n"                     \
  61         : "=&r"(temp)   /* Early clobber, to prevent reg reuse */       \
  62         : "r"(m),       /* Not "m": llock only supports reg direct addr mode */ \
  63           "ir"(nr)                                                      \
  64         : "cc");                                                        \
  65 }
  66
  67 /*
  68  * Semantically:
  69  *    Test the bit
  70  *    if clear
  71  *        set it and return 0 (old value)
  72  *    else
  73  *        return 1 (old value).
  74  *
  75  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
  76  * and the old value of bit is returned
  77  */
  78 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
  79 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
  80 {                                                                       \
  81         unsigned long old, temp;                                        \
  82                                                                         \
  83         m += nr >> 5;                                                   \
  84                                                                         \
  85         if (__builtin_constant_p(nr))                                   \
  86                 nr &= 0x1f;                                             \
  87                                                                         \
  88         /*                                                              \
  89          * Explicit full memory barrier needed before/after as          \
  90          * LLOCK/SCOND themselves don't provide any such smenatic       \
  91          */                                                             \
  92         smp_mb();                                                       \
  93                                                                         \
  94         __asm__ __volatile__(                                           \
  95         "1:     llock       %0, [%2]    \n"                             \
  96         "       " #asm_op " %1, %0, %3  \n"                             \
  97         "       scond       %1, [%2]    \n"                             \
  98         "       bnz         1b          \n"                             \
  99         : "=&r"(old), "=&r"(temp)                                       \
 100         : "r"(m), "ir"(nr)                                              \
 101         : "cc");                                                        \
 102                                                                         \
 103         smp_mb();                                                       \
 104                                                                         \
 105         return (old & (1 << nr)) != 0;                                  \
 106 }
 107
 108 #else   /* !CONFIG_ARC_HAS_LLSC */
 109
 110 /*
 111  * Non hardware assisted Atomic-R-M-W
 112  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 113  *
 114  * There's "significant" micro-optimization in writing our own variants of
 115  * bitops (over generic variants)
 116  *
 117  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
 118  *     This avoids extra code to be generated for pointer arithmatic, since
 119  *     is "not sure" that index is NOT -ve
 120  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
 121  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
 122  *     (GCC Quirk: however for constant @nr we still need to do the masking
 123  *             at compile time)
 124  */
 125
 126 #define BIT_OP(op, c_op, asm_op)                                        \
 127 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
 128 {                                                                       \
 129         unsigned long temp, flags;                                      \
 130         m += nr >> 5;                                                   \
 131                                                                         \
 132         if (__builtin_constant_p(nr))                                   \
 133                 nr &= 0x1f;                                             \
 134                                                                         \
 135         /*                                                              \
 136          * spin lock/unlock provide the needed smp_mb() before/after    \
 137          */                                                             \
 138         bitops_lock(flags);                                             \
 139                                                                         \
 140         temp = *m;                                                      \
 141         *m = temp c_op (1UL << nr);                                     \
 142                                                                         \
 143         bitops_unlock(flags);                                           \
 144 }
 145
 146 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 147 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 148 {                                                                       \
 149         unsigned long old, flags;                                       \
 150         m += nr >> 5;                                                   \
 151                                                                         \
 152         if (__builtin_constant_p(nr))                                   \
 153                 nr &= 0x1f;                                             \
 154                                                                         \
 155         bitops_lock(flags);                                             \
 156                                                                         \
 157         old = *m;                                                       \
 158         *m = old c_op (1 << nr);                                        \
 159                                                                         \
 160         bitops_unlock(flags);                                           \
 161                                                                         \
 162         return (old & (1 << nr)) != 0;                                  \
 163 }
 164
 165 #endif /* CONFIG_ARC_HAS_LLSC */
 166
 167 /***************************************
 168  * Non atomic variants
 169  **************************************/
 170
 171 #define __BIT_OP(op, c_op, asm_op)                                      \
 172 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m)    \
 173 {                                                                       \
 174         unsigned long temp;                                             \
 175         m += nr >> 5;                                                   \
 176                                                                         \
 177         if (__builtin_constant_p(nr))                                   \
 178                 nr &= 0x1f;                                             \
 179                                                                         \
 180         temp = *m;                                                      \
 181         *m = temp c_op (1UL << nr);                                     \
 182 }
 183
 184 #define __TEST_N_BIT_OP(op, c_op, asm_op)                               \
 185 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 186 {                                                                       \
 187         unsigned long old;                                              \
 188         m += nr >> 5;                                                   \
 189                                                                         \
 190         if (__builtin_constant_p(nr))                                   \
 191                 nr &= 0x1f;                                             \
 192                                                                         \
 193         old = *m;                                                       \
 194         *m = old c_op (1 << nr);                                        \
 195                                                                         \
 196         return (old & (1 << nr)) != 0;                                  \
 197 }
 198
 199 #define BIT_OPS(op, c_op, asm_op)                                       \
 200                                                                         \
 201         /* set_bit(), clear_bit(), change_bit() */                      \
 202         BIT_OP(op, c_op, asm_op)                                        \
 203                                                                         \
 204         /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\
 205         TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 206                                                                         \
 207         /* __set_bit(), __clear_bit(), __change_bit() */                \
 208         __BIT_OP(op, c_op, asm_op)                                      \
 209                                                                         \
 210         /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\
 211         __TEST_N_BIT_OP(op, c_op, asm_op)
 212
 213 BIT_OPS(set, |, bset)
 214 BIT_OPS(clear, & ~, bclr)
 215 BIT_OPS(change, ^, bxor)
 216
 217 /*
 218  * This routine doesn't need to be atomic.
 219  */
 220 static inline int
 221 test_bit(unsigned int nr, const volatile unsigned long *addr)
 222 {
 223         unsigned long mask;
 224
 225         addr += nr >> 5;
 226
 227         if (__builtin_constant_p(nr))
 228                 nr &= 0x1f;
 229
 230         mask = 1 << nr;
 231
 232         return ((mask & *addr) != 0);
 233 }
 234
 235 #ifdef CONFIG_ISA_ARCOMPACT
 236
 237 /*
 238  * Count the number of zeros, starting from MSB
 239  * Helper for fls( ) friends
 240  * This is a pure count, so (1-32) or (0-31) doesn't apply
 241  * It could be 0 to 32, based on num of 0's in there
 242  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
 243  */
 244 static inline __attribute__ ((const)) int clz(unsigned int x)
 245 {
 246         unsigned int res;
 247
 248         __asm__ __volatile__(
 249         "       norm.f  %0, %1          \n"
 250         "       mov.n   %0, 0           \n"
 251         "       add.p   %0, %0, 1       \n"
 252         : "=r"(res)
 253         : "r"(x)
 254         : "cc");
 255
 256         return res;
 257 }
 258
 259 static inline int constant_fls(int x)
 260 {
 261         int r = 32;
 262
 263         if (!x)
 264                 return 0;
 265         if (!(x & 0xffff0000u)) {
 266                 x <<= 16;
 267                 r -= 16;
 268         }
 269         if (!(x & 0xff000000u)) {
 270                 x <<= 8;
 271                 r -= 8;
 272         }
 273         if (!(x & 0xf0000000u)) {
 274                 x <<= 4;
 275                 r -= 4;
 276         }
 277         if (!(x & 0xc0000000u)) {
 278                 x <<= 2;
 279                 r -= 2;
 280         }
 281         if (!(x & 0x80000000u)) {
 282                 x <<= 1;
 283                 r -= 1;
 284         }
 285         return r;
 286 }
 287
 288 /*
 289  * fls = Find Last Set in word
 290  * @result: [1-32]
 291  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 292  */
 293 static inline __attribute__ ((const)) int fls(unsigned long x)
 294 {
 295         if (__builtin_constant_p(x))
 296                return constant_fls(x);
 297
 298         return 32 - clz(x);
 299 }
 300
 301 /*
 302  * __fls: Similar to fls, but zero based (0-31)
 303  */
 304 static inline __attribute__ ((const)) int __fls(unsigned long x)
 305 {
 306         if (!x)
 307                 return 0;
 308         else
 309                 return fls(x) - 1;
 310 }
 311
 312 /*
 313  * ffs = Find First Set in word (LSB to MSB)
 314  * @result: [1-32], 0 if all 0's
 315  */
 316 #define ffs(x)  ({ unsigned long __t = (x); fls(__t & -__t); })
 317
 318 /*
 319  * __ffs: Similar to ffs, but zero based (0-31)
 320  */
 321 static inline __attribute__ ((const)) int __ffs(unsigned long word)
 322 {
 323         if (!word)
 324                 return word;
 325
 326         return ffs(word) - 1;
 327 }
 328
 329 #else   /* CONFIG_ISA_ARCV2 */
 330
 331 /*
 332  * fls = Find Last Set in word
 333  * @result: [1-32]
 334  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 335  */
 336 static inline __attribute__ ((const)) int fls(unsigned long x)
 337 {
 338         int n;
 339
 340         asm volatile(
 341         "       fls.f   %0, %1          \n"  /* 0:31; 0(Z) if src 0 */
 342         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 343         : "=r"(n)       /* Early clobber not needed */
 344         : "r"(x)
 345         : "cc");
 346
 347         return n;
 348 }
 349
 350 /*
 351  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
 352  */
 353 static inline __attribute__ ((const)) int __fls(unsigned long x)
 354 {
 355         /* FLS insn has exactly same semantics as the API */
 356         return  __builtin_arc_fls(x);
 357 }
 358
 359 /*
 360  * ffs = Find First Set in word (LSB to MSB)
 361  * @result: [1-32], 0 if all 0's
 362  */
 363 static inline __attribute__ ((const)) int ffs(unsigned long x)
 364 {
 365         int n;
 366
 367         asm volatile(
 368         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 369         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 370         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 371         : "=r"(n)       /* Early clobber not needed */
 372         : "r"(x)
 373         : "cc");
 374
 375         return n;
 376 }
 377
 378 /*
 379  * __ffs: Similar to ffs, but zero based (0-31)
 380  */
 381 static inline __attribute__ ((const)) int __ffs(unsigned long x)
 382 {
 383         int n;
 384
 385         asm volatile(
 386         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 387         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 388         : "=r"(n)
 389         : "r"(x)
 390         : "cc");
 391
 392         return n;
 393
 394 }
 395
 396 #endif  /* CONFIG_ISA_ARCOMPACT */
 397
 398 /*
 399  * ffz = Find First Zero in word.
 400  * @return:[0-31], 32 if all 1's
 401  */
 402 #define ffz(x)  __ffs(~(x))
 403
 404 #include <asm-generic/bitops/hweight.h>
 405 #include <asm-generic/bitops/fls64.h>
 406 #include <asm-generic/bitops/sched.h>
 407 #include <asm-generic/bitops/lock.h>
 408
 409 #include <asm-generic/bitops/find.h>
 410 #include <asm-generic/bitops/le.h>
 411 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 412
 413 #endif /* !__ASSEMBLY__ */
 414
 415 #endif