arch/mips/include/asm/spinlock.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  */
   9 #ifndef _ASM_SPINLOCK_H
  10 #define _ASM_SPINLOCK_H
  11
  12 #include <linux/compiler.h>
  13
  14 #include <asm/barrier.h>
  15 #include <asm/compiler.h>
  16 #include <asm/war.h>
  17
  18 /*
  19  * Your basic SMP spinlocks, allowing only a single CPU anywhere
  20  *
  21  * Simple spin lock operations.  There are two variants, one clears IRQ's
  22  * on the local processor, one does not.
  23  *
  24  * These are fair FIFO ticket locks
  25  *
  26  * (the type definitions are in asm/spinlock_types.h)
  27  */
  28
  29
  30 /*
  31  * Ticket locks are conceptually two parts, one indicating the current head of
  32  * the queue, and the other indicating the current tail. The lock is acquired
  33  * by atomically noting the tail and incrementing it by one (thus adding
  34  * ourself to the queue and noting our position), then waiting until the head
  35  * becomes equal to the the initial value of the tail.
  36  */
  37
  38 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
  39 {
  40         u32 counters = ACCESS_ONCE(lock->lock);
  41
  42         return ((counters >> 16) ^ counters) & 0xffff;
  43 }
  44
  45 static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
  46 {
  47         return lock.h.serving_now == lock.h.ticket;
  48 }
  49
  50 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
  51 #define arch_spin_unlock_wait(x) \
  52         while (arch_spin_is_locked(x)) { cpu_relax(); }
  53
  54 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
  55 {
  56         u32 counters = ACCESS_ONCE(lock->lock);
  57
  58         return (((counters >> 16) - counters) & 0xffff) > 1;
  59 }
  60 #define arch_spin_is_contended  arch_spin_is_contended
  61
  62 static inline void arch_spin_lock(arch_spinlock_t *lock)
  63 {
  64         int my_ticket;
  65         int tmp;
  66         int inc = 0x10000;
  67
  68         if (R10000_LLSC_WAR) {
  69                 __asm__ __volatile__ (
  70                 "       .set push               # arch_spin_lock        \n"
  71                 "       .set noreorder                                  \n"
  72                 "                                                       \n"
  73                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
  74                 "       addu    %[my_ticket], %[ticket], %[inc]         \n"
  75                 "       sc      %[my_ticket], %[ticket_ptr]             \n"
  76                 "       beqzl   %[my_ticket], 1b                        \n"
  77                 "        nop                                            \n"
  78                 "       srl     %[my_ticket], %[ticket], 16             \n"
  79                 "       andi    %[ticket], %[ticket], 0xffff            \n"
  80                 "       bne     %[ticket], %[my_ticket], 4f             \n"
  81                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
  82                 "2:                                                     \n"
  83                 "       .subsection 2                                   \n"
  84                 "4:     andi    %[ticket], %[ticket], 0xffff            \n"
  85                 "       sll     %[ticket], 5                            \n"
  86                 "                                                       \n"
  87                 "6:     bnez    %[ticket], 6b                           \n"
  88                 "        subu   %[ticket], 1                            \n"
  89                 "                                                       \n"
  90                 "       lhu     %[ticket], %[serving_now_ptr]           \n"
  91                 "       beq     %[ticket], %[my_ticket], 2b             \n"
  92                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
  93                 "       b       4b                                      \n"
  94                 "        subu   %[ticket], %[ticket], 1                 \n"
  95                 "       .previous                                       \n"
  96                 "       .set pop                                        \n"
  97                 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
  98                   [serving_now_ptr] "+m" (lock->h.serving_now),
  99                   [ticket] "=&r" (tmp),
 100                   [my_ticket] "=&r" (my_ticket)
 101                 : [inc] "r" (inc));
 102         } else {
 103                 __asm__ __volatile__ (
 104                 "       .set push               # arch_spin_lock        \n"
 105                 "       .set noreorder                                  \n"
 106                 "                                                       \n"
 107                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 108                 "       addu    %[my_ticket], %[ticket], %[inc]         \n"
 109                 "       sc      %[my_ticket], %[ticket_ptr]             \n"
 110                 "       beqz    %[my_ticket], 1b                        \n"
 111                 "        srl    %[my_ticket], %[ticket], 16             \n"
 112                 "       andi    %[ticket], %[ticket], 0xffff            \n"
 113                 "       bne     %[ticket], %[my_ticket], 4f             \n"
 114                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
 115                 "2:                                                     \n"
 116                 "       .subsection 2                                   \n"
 117                 "4:     andi    %[ticket], %[ticket], 0xffff            \n"
 118                 "       sll     %[ticket], 5                            \n"
 119                 "                                                       \n"
 120                 "6:     bnez    %[ticket], 6b                           \n"
 121                 "        subu   %[ticket], 1                            \n"
 122                 "                                                       \n"
 123                 "       lhu     %[ticket], %[serving_now_ptr]           \n"
 124                 "       beq     %[ticket], %[my_ticket], 2b             \n"
 125                 "        subu   %[ticket], %[my_ticket], %[ticket]      \n"
 126                 "       b       4b                                      \n"
 127                 "        subu   %[ticket], %[ticket], 1                 \n"
 128                 "       .previous                                       \n"
 129                 "       .set pop                                        \n"
 130                 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
 131                   [serving_now_ptr] "+m" (lock->h.serving_now),
 132                   [ticket] "=&r" (tmp),
 133                   [my_ticket] "=&r" (my_ticket)
 134                 : [inc] "r" (inc));
 135         }
 136
 137         smp_llsc_mb();
 138 }
 139
 140 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 141 {
 142         unsigned int serving_now = lock->h.serving_now + 1;
 143         wmb();
 144         lock->h.serving_now = (u16)serving_now;
 145         nudge_writes();
 146 }
 147
 148 static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
 149 {
 150         int tmp, tmp2, tmp3;
 151         int inc = 0x10000;
 152
 153         if (R10000_LLSC_WAR) {
 154                 __asm__ __volatile__ (
 155                 "       .set push               # arch_spin_trylock     \n"
 156                 "       .set noreorder                                  \n"
 157                 "                                                       \n"
 158                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 159                 "       srl     %[my_ticket], %[ticket], 16             \n"
 160                 "       andi    %[now_serving], %[ticket], 0xffff       \n"
 161                 "       bne     %[my_ticket], %[now_serving], 3f        \n"
 162                 "        addu   %[ticket], %[ticket], %[inc]            \n"
 163                 "       sc      %[ticket], %[ticket_ptr]                \n"
 164                 "       beqzl   %[ticket], 1b                           \n"
 165                 "        li     %[ticket], 1                            \n"
 166                 "2:                                                     \n"
 167                 "       .subsection 2                                   \n"
 168                 "3:     b       2b                                      \n"
 169                 "        li     %[ticket], 0                            \n"
 170                 "       .previous                                       \n"
 171                 "       .set pop                                        \n"
 172                 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
 173                   [ticket] "=&r" (tmp),
 174                   [my_ticket] "=&r" (tmp2),
 175                   [now_serving] "=&r" (tmp3)
 176                 : [inc] "r" (inc));
 177         } else {
 178                 __asm__ __volatile__ (
 179                 "       .set push               # arch_spin_trylock     \n"
 180                 "       .set noreorder                                  \n"
 181                 "                                                       \n"
 182                 "1:     ll      %[ticket], %[ticket_ptr]                \n"
 183                 "       srl     %[my_ticket], %[ticket], 16             \n"
 184                 "       andi    %[now_serving], %[ticket], 0xffff       \n"
 185                 "       bne     %[my_ticket], %[now_serving], 3f        \n"
 186                 "        addu   %[ticket], %[ticket], %[inc]            \n"
 187                 "       sc      %[ticket], %[ticket_ptr]                \n"
 188                 "       beqz    %[ticket], 1b                           \n"
 189                 "        li     %[ticket], 1                            \n"
 190                 "2:                                                     \n"
 191                 "       .subsection 2                                   \n"
 192                 "3:     b       2b                                      \n"
 193                 "        li     %[ticket], 0                            \n"
 194                 "       .previous                                       \n"
 195                 "       .set pop                                        \n"
 196                 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
 197                   [ticket] "=&r" (tmp),
 198                   [my_ticket] "=&r" (tmp2),
 199                   [now_serving] "=&r" (tmp3)
 200                 : [inc] "r" (inc));
 201         }
 202
 203         smp_llsc_mb();
 204
 205         return tmp;
 206 }
 207
 208 /*
 209  * Read-write spinlocks, allowing multiple readers but only one writer.
 210  *
 211  * NOTE! it is quite common to have readers in interrupts but no interrupt
 212  * writers. For those circumstances we can "mix" irq-safe locks - any writer
 213  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 214  * read-locks.
 215  */
 216
 217 /*
 218  * read_can_lock - would read_trylock() succeed?
 219  * @lock: the rwlock in question.
 220  */
 221 #define arch_read_can_lock(rw)  ((rw)->lock >= 0)
 222
 223 /*
 224  * write_can_lock - would write_trylock() succeed?
 225  * @lock: the rwlock in question.
 226  */
 227 #define arch_write_can_lock(rw) (!(rw)->lock)
 228
 229 static inline void arch_read_lock(arch_rwlock_t *rw)
 230 {
 231         unsigned int tmp;
 232
 233         if (R10000_LLSC_WAR) {
 234                 __asm__ __volatile__(
 235                 "       .set    noreorder       # arch_read_lock        \n"
 236                 "1:     ll      %1, %2                                  \n"
 237                 "       bltz    %1, 1b                                  \n"
 238                 "        addu   %1, 1                                   \n"
 239                 "       sc      %1, %0                                  \n"
 240                 "       beqzl   %1, 1b                                  \n"
 241                 "        nop                                            \n"
 242                 "       .set    reorder                                 \n"
 243                 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
 244                 : GCC_OFF_SMALL_ASM() (rw->lock)
 245                 : "memory");
 246         } else {
 247                 do {
 248                         __asm__ __volatile__(
 249                         "1:     ll      %1, %2  # arch_read_lock        \n"
 250                         "       bltz    %1, 1b                          \n"
 251                         "        addu   %1, 1                           \n"
 252                         "2:     sc      %1, %0                          \n"
 253                         : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
 254                         : GCC_OFF_SMALL_ASM() (rw->lock)
 255                         : "memory");
 256                 } while (unlikely(!tmp));
 257         }
 258
 259         smp_llsc_mb();
 260 }
 261
 262 static inline void arch_read_unlock(arch_rwlock_t *rw)
 263 {
 264         unsigned int tmp;
 265
 266         smp_mb__before_llsc();
 267
 268         if (R10000_LLSC_WAR) {
 269                 __asm__ __volatile__(
 270                 "1:     ll      %1, %2          # arch_read_unlock      \n"
 271                 "       addiu   %1, -1                                  \n"
 272                 "       sc      %1, %0                                  \n"
 273                 "       beqzl   %1, 1b                                  \n"
 274                 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
 275                 : GCC_OFF_SMALL_ASM() (rw->lock)
 276                 : "memory");
 277         } else {
 278                 do {
 279                         __asm__ __volatile__(
 280                         "1:     ll      %1, %2  # arch_read_unlock      \n"
 281                         "       addiu   %1, -1                          \n"
 282                         "       sc      %1, %0                          \n"
 283                         : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
 284                         : GCC_OFF_SMALL_ASM() (rw->lock)
 285                         : "memory");
 286                 } while (unlikely(!tmp));
 287         }
 288 }
 289
 290 static inline void arch_write_lock(arch_rwlock_t *rw)
 291 {
 292         unsigned int tmp;
 293
 294         if (R10000_LLSC_WAR) {
 295                 __asm__ __volatile__(
 296                 "       .set    noreorder       # arch_write_lock       \n"
 297                 "1:     ll      %1, %2                                  \n"
 298                 "       bnez    %1, 1b                                  \n"
 299                 "        lui    %1, 0x8000                              \n"
 300                 "       sc      %1, %0                                  \n"
 301                 "       beqzl   %1, 1b                                  \n"
 302                 "        nop                                            \n"
 303                 "       .set    reorder                                 \n"
 304                 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
 305                 : GCC_OFF_SMALL_ASM() (rw->lock)
 306                 : "memory");
 307         } else {
 308                 do {
 309                         __asm__ __volatile__(
 310                         "1:     ll      %1, %2  # arch_write_lock       \n"
 311                         "       bnez    %1, 1b                          \n"
 312                         "        lui    %1, 0x8000                      \n"
 313                         "2:     sc      %1, %0                          \n"
 314                         : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
 315                         : GCC_OFF_SMALL_ASM() (rw->lock)
 316                         : "memory");
 317                 } while (unlikely(!tmp));
 318         }
 319
 320         smp_llsc_mb();
 321 }
 322
 323 static inline void arch_write_unlock(arch_rwlock_t *rw)
 324 {
 325         smp_mb__before_llsc();
 326
 327         __asm__ __volatile__(
 328         "                               # arch_write_unlock     \n"
 329         "       sw      $0, %0                                  \n"
 330         : "=m" (rw->lock)
 331         : "m" (rw->lock)
 332         : "memory");
 333 }
 334
 335 static inline int arch_read_trylock(arch_rwlock_t *rw)
 336 {
 337         unsigned int tmp;
 338         int ret;
 339
 340         if (R10000_LLSC_WAR) {
 341                 __asm__ __volatile__(
 342                 "       .set    noreorder       # arch_read_trylock     \n"
 343                 "       li      %2, 0                                   \n"
 344                 "1:     ll      %1, %3                                  \n"
 345                 "       bltz    %1, 2f                                  \n"
 346                 "        addu   %1, 1                                   \n"
 347                 "       sc      %1, %0                                  \n"
 348                 "       .set    reorder                                 \n"
 349                 "       beqzl   %1, 1b                                  \n"
 350                 "        nop                                            \n"
 351                 __WEAK_LLSC_MB
 352                 "       li      %2, 1                                   \n"
 353                 "2:                                                     \n"
 354                 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
 355                 : GCC_OFF_SMALL_ASM() (rw->lock)
 356                 : "memory");
 357         } else {
 358                 __asm__ __volatile__(
 359                 "       .set    noreorder       # arch_read_trylock     \n"
 360                 "       li      %2, 0                                   \n"
 361                 "1:     ll      %1, %3                                  \n"
 362                 "       bltz    %1, 2f                                  \n"
 363                 "        addu   %1, 1                                   \n"
 364                 "       sc      %1, %0                                  \n"
 365                 "       beqz    %1, 1b                                  \n"
 366                 "        nop                                            \n"
 367                 "       .set    reorder                                 \n"
 368                 __WEAK_LLSC_MB
 369                 "       li      %2, 1                                   \n"
 370                 "2:                                                     \n"
 371                 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
 372                 : GCC_OFF_SMALL_ASM() (rw->lock)
 373                 : "memory");
 374         }
 375
 376         return ret;
 377 }
 378
 379 static inline int arch_write_trylock(arch_rwlock_t *rw)
 380 {
 381         unsigned int tmp;
 382         int ret;
 383
 384         if (R10000_LLSC_WAR) {
 385                 __asm__ __volatile__(
 386                 "       .set    noreorder       # arch_write_trylock    \n"
 387                 "       li      %2, 0                                   \n"
 388                 "1:     ll      %1, %3                                  \n"
 389                 "       bnez    %1, 2f                                  \n"
 390                 "        lui    %1, 0x8000                              \n"
 391                 "       sc      %1, %0                                  \n"
 392                 "       beqzl   %1, 1b                                  \n"
 393                 "        nop                                            \n"
 394                 __WEAK_LLSC_MB
 395                 "       li      %2, 1                                   \n"
 396                 "       .set    reorder                                 \n"
 397                 "2:                                                     \n"
 398                 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
 399                 : GCC_OFF_SMALL_ASM() (rw->lock)
 400                 : "memory");
 401         } else {
 402                 do {
 403                         __asm__ __volatile__(
 404                         "       ll      %1, %3  # arch_write_trylock    \n"
 405                         "       li      %2, 0                           \n"
 406                         "       bnez    %1, 2f                          \n"
 407                         "       lui     %1, 0x8000                      \n"
 408                         "       sc      %1, %0                          \n"
 409                         "       li      %2, 1                           \n"
 410                         "2:                                             \n"
 411                         : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
 412                           "=&r" (ret)
 413                         : GCC_OFF_SMALL_ASM() (rw->lock)
 414                         : "memory");
 415                 } while (unlikely(!tmp));
 416
 417                 smp_llsc_mb();
 418         }
 419
 420         return ret;
 421 }
 422
 423 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
 424 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
 425
 426 #define arch_spin_relax(lock)   cpu_relax()
 427 #define arch_read_relax(lock)   cpu_relax()
 428 #define arch_write_relax(lock)  cpu_relax()
 429
 430 #endif /* _ASM_SPINLOCK_H */