drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38 #include <linux/dma-buf.h>
  39
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  42                                                    bool force);
  43 static __must_check int
  44 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  45                                bool readonly);
  46 static __must_check int
  47 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
  48                            struct i915_address_space *vm,
  49                            unsigned alignment,
  50                            bool map_and_fenceable,
  51                            bool nonblocking);
  52 static int i915_gem_phys_pwrite(struct drm_device *dev,
  53                                 struct drm_i915_gem_object *obj,
  54                                 struct drm_i915_gem_pwrite *args,
  55                                 struct drm_file *file);
  56
  57 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  58                                  struct drm_i915_gem_object *obj);
  59 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  60                                          struct drm_i915_fence_reg *fence,
  61                                          bool enable);
  62
  63 static unsigned long i915_gem_inactive_count(struct shrinker *shrinker,
  64                                              struct shrink_control *sc);
  65 static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker,
  66                                             struct shrink_control *sc);
  67 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  68 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  69 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  70
  71 static bool cpu_cache_is_coherent(struct drm_device *dev,
  72                                   enum i915_cache_level level)
  73 {
  74         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  75 }
  76
  77 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  78 {
  79         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  80                 return true;
  81
  82         return obj->pin_display;
  83 }
  84
  85 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  86 {
  87         if (obj->tiling_mode)
  88                 i915_gem_release_mmap(obj);
  89
  90         /* As we do not have an associated fence register, we will force
  91          * a tiling change if we ever need to acquire one.
  92          */
  93         obj->fence_dirty = false;
  94         obj->fence_reg = I915_FENCE_REG_NONE;
  95 }
  96
  97 /* some bookkeeping */
  98 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  99                                   size_t size)
 100 {
 101         spin_lock(&dev_priv->mm.object_stat_lock);
 102         dev_priv->mm.object_count++;
 103         dev_priv->mm.object_memory += size;
 104         spin_unlock(&dev_priv->mm.object_stat_lock);
 105 }
 106
 107 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 108                                      size_t size)
 109 {
 110         spin_lock(&dev_priv->mm.object_stat_lock);
 111         dev_priv->mm.object_count--;
 112         dev_priv->mm.object_memory -= size;
 113         spin_unlock(&dev_priv->mm.object_stat_lock);
 114 }
 115
 116 static int
 117 i915_gem_wait_for_error(struct i915_gpu_error *error)
 118 {
 119         int ret;
 120
 121 #define EXIT_COND (!i915_reset_in_progress(error) || \
 122                    i915_terminally_wedged(error))
 123         if (EXIT_COND)
 124                 return 0;
 125
 126         /*
 127          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 128          * userspace. If it takes that long something really bad is going on and
 129          * we should simply try to bail out and fail as gracefully as possible.
 130          */
 131         ret = wait_event_interruptible_timeout(error->reset_queue,
 132                                                EXIT_COND,
 133                                                10*HZ);
 134         if (ret == 0) {
 135                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 136                 return -EIO;
 137         } else if (ret < 0) {
 138                 return ret;
 139         }
 140 #undef EXIT_COND
 141
 142         return 0;
 143 }
 144
 145 int i915_mutex_lock_interruptible(struct drm_device *dev)
 146 {
 147         struct drm_i915_private *dev_priv = dev->dev_private;
 148         int ret;
 149
 150         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 151         if (ret)
 152                 return ret;
 153
 154         ret = mutex_lock_interruptible(&dev->struct_mutex);
 155         if (ret)
 156                 return ret;
 157
 158         WARN_ON(i915_verify_lists(dev));
 159         return 0;
 160 }
 161
 162 static inline bool
 163 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 164 {
 165         return i915_gem_obj_bound_any(obj) && !obj->active;
 166 }
 167
 168 int
 169 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 170                     struct drm_file *file)
 171 {
 172         struct drm_i915_private *dev_priv = dev->dev_private;
 173         struct drm_i915_gem_init *args = data;
 174
 175         if (drm_core_check_feature(dev, DRIVER_MODESET))
 176                 return -ENODEV;
 177
 178         if (args->gtt_start >= args->gtt_end ||
 179             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 180                 return -EINVAL;
 181
 182         /* GEM with user mode setting was never supported on ilk and later. */
 183         if (INTEL_INFO(dev)->gen >= 5)
 184                 return -ENODEV;
 185
 186         mutex_lock(&dev->struct_mutex);
 187         i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
 188                                   args->gtt_end);
 189         dev_priv->gtt.mappable_end = args->gtt_end;
 190         mutex_unlock(&dev->struct_mutex);
 191
 192         return 0;
 193 }
 194
 195 int
 196 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 197                             struct drm_file *file)
 198 {
 199         struct drm_i915_private *dev_priv = dev->dev_private;
 200         struct drm_i915_gem_get_aperture *args = data;
 201         struct drm_i915_gem_object *obj;
 202         size_t pinned;
 203
 204         pinned = 0;
 205         mutex_lock(&dev->struct_mutex);
 206         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
 207                 if (obj->pin_count)
 208                         pinned += i915_gem_obj_ggtt_size(obj);
 209         mutex_unlock(&dev->struct_mutex);
 210
 211         args->aper_size = dev_priv->gtt.base.total;
 212         args->aper_available_size = args->aper_size - pinned;
 213
 214         return 0;
 215 }
 216
 217 void *i915_gem_object_alloc(struct drm_device *dev)
 218 {
 219         struct drm_i915_private *dev_priv = dev->dev_private;
 220         return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
 221 }
 222
 223 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 224 {
 225         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 226         kmem_cache_free(dev_priv->slab, obj);
 227 }
 228
 229 static int
 230 i915_gem_create(struct drm_file *file,
 231                 struct drm_device *dev,
 232                 uint64_t size,
 233                 uint32_t *handle_p)
 234 {
 235         struct drm_i915_gem_object *obj;
 236         int ret;
 237         u32 handle;
 238
 239         size = roundup(size, PAGE_SIZE);
 240         if (size == 0)
 241                 return -EINVAL;
 242
 243         /* Allocate the new object */
 244         obj = i915_gem_alloc_object(dev, size);
 245         if (obj == NULL)
 246                 return -ENOMEM;
 247
 248         ret = drm_gem_handle_create(file, &obj->base, &handle);
 249         /* drop reference from allocate - handle holds it now */
 250         drm_gem_object_unreference_unlocked(&obj->base);
 251         if (ret)
 252                 return ret;
 253
 254         *handle_p = handle;
 255         return 0;
 256 }
 257
 258 int
 259 i915_gem_dumb_create(struct drm_file *file,
 260                      struct drm_device *dev,
 261                      struct drm_mode_create_dumb *args)
 262 {
 263         /* have to work out size/pitch and return them */
 264         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 265         args->size = args->pitch * args->height;
 266         return i915_gem_create(file, dev,
 267                                args->size, &args->handle);
 268 }
 269
 270 /**
 271  * Creates a new mm object and returns a handle to it.
 272  */
 273 int
 274 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 275                       struct drm_file *file)
 276 {
 277         struct drm_i915_gem_create *args = data;
 278
 279         return i915_gem_create(file, dev,
 280                                args->size, &args->handle);
 281 }
 282
 283 static inline int
 284 __copy_to_user_swizzled(char __user *cpu_vaddr,
 285                         const char *gpu_vaddr, int gpu_offset,
 286                         int length)
 287 {
 288         int ret, cpu_offset = 0;
 289
 290         while (length > 0) {
 291                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 292                 int this_length = min(cacheline_end - gpu_offset, length);
 293                 int swizzled_gpu_offset = gpu_offset ^ 64;
 294
 295                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 296                                      gpu_vaddr + swizzled_gpu_offset,
 297                                      this_length);
 298                 if (ret)
 299                         return ret + length;
 300
 301                 cpu_offset += this_length;
 302                 gpu_offset += this_length;
 303                 length -= this_length;
 304         }
 305
 306         return 0;
 307 }
 308
 309 static inline int
 310 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 311                           const char __user *cpu_vaddr,
 312                           int length)
 313 {
 314         int ret, cpu_offset = 0;
 315
 316         while (length > 0) {
 317                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 318                 int this_length = min(cacheline_end - gpu_offset, length);
 319                 int swizzled_gpu_offset = gpu_offset ^ 64;
 320
 321                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 322                                        cpu_vaddr + cpu_offset,
 323                                        this_length);
 324                 if (ret)
 325                         return ret + length;
 326
 327                 cpu_offset += this_length;
 328                 gpu_offset += this_length;
 329                 length -= this_length;
 330         }
 331
 332         return 0;
 333 }
 334
 335 /* Per-page copy function for the shmem pread fastpath.
 336  * Flushes invalid cachelines before reading the target if
 337  * needs_clflush is set. */
 338 static int
 339 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 340                  char __user *user_data,
 341                  bool page_do_bit17_swizzling, bool needs_clflush)
 342 {
 343         char *vaddr;
 344         int ret;
 345
 346         if (unlikely(page_do_bit17_swizzling))
 347                 return -EINVAL;
 348
 349         vaddr = kmap_atomic(page);
 350         if (needs_clflush)
 351                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 352                                        page_length);
 353         ret = __copy_to_user_inatomic(user_data,
 354                                       vaddr + shmem_page_offset,
 355                                       page_length);
 356         kunmap_atomic(vaddr);
 357
 358         return ret ? -EFAULT : 0;
 359 }
 360
 361 static void
 362 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 363                              bool swizzled)
 364 {
 365         if (unlikely(swizzled)) {
 366                 unsigned long start = (unsigned long) addr;
 367                 unsigned long end = (unsigned long) addr + length;
 368
 369                 /* For swizzling simply ensure that we always flush both
 370                  * channels. Lame, but simple and it works. Swizzled
 371                  * pwrite/pread is far from a hotpath - current userspace
 372                  * doesn't use it at all. */
 373                 start = round_down(start, 128);
 374                 end = round_up(end, 128);
 375
 376                 drm_clflush_virt_range((void *)start, end - start);
 377         } else {
 378                 drm_clflush_virt_range(addr, length);
 379         }
 380
 381 }
 382
 383 /* Only difference to the fast-path function is that this can handle bit17
 384  * and uses non-atomic copy and kmap functions. */
 385 static int
 386 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 387                  char __user *user_data,
 388                  bool page_do_bit17_swizzling, bool needs_clflush)
 389 {
 390         char *vaddr;
 391         int ret;
 392
 393         vaddr = kmap(page);
 394         if (needs_clflush)
 395                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 396                                              page_length,
 397                                              page_do_bit17_swizzling);
 398
 399         if (page_do_bit17_swizzling)
 400                 ret = __copy_to_user_swizzled(user_data,
 401                                               vaddr, shmem_page_offset,
 402                                               page_length);
 403         else
 404                 ret = __copy_to_user(user_data,
 405                                      vaddr + shmem_page_offset,
 406                                      page_length);
 407         kunmap(page);
 408
 409         return ret ? - EFAULT : 0;
 410 }
 411
 412 static int
 413 i915_gem_shmem_pread(struct drm_device *dev,
 414                      struct drm_i915_gem_object *obj,
 415                      struct drm_i915_gem_pread *args,
 416                      struct drm_file *file)
 417 {
 418         char __user *user_data;
 419         ssize_t remain;
 420         loff_t offset;
 421         int shmem_page_offset, page_length, ret = 0;
 422         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 423         int prefaulted = 0;
 424         int needs_clflush = 0;
 425         struct sg_page_iter sg_iter;
 426
 427         user_data = to_user_ptr(args->data_ptr);
 428         remain = args->size;
 429
 430         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 431
 432         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 433                 /* If we're not in the cpu read domain, set ourself into the gtt
 434                  * read domain and manually flush cachelines (if required). This
 435                  * optimizes for the case when the gpu will dirty the data
 436                  * anyway again before the next pread happens. */
 437                 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
 438                 ret = i915_gem_object_wait_rendering(obj, true);
 439                 if (ret)
 440                         return ret;
 441         }
 442
 443         ret = i915_gem_object_get_pages(obj);
 444         if (ret)
 445                 return ret;
 446
 447         i915_gem_object_pin_pages(obj);
 448
 449         offset = args->offset;
 450
 451         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 452                          offset >> PAGE_SHIFT) {
 453                 struct page *page = sg_page_iter_page(&sg_iter);
 454
 455                 if (remain <= 0)
 456                         break;
 457
 458                 /* Operation in this page
 459                  *
 460                  * shmem_page_offset = offset within page in shmem file
 461                  * page_length = bytes to copy for this page
 462                  */
 463                 shmem_page_offset = offset_in_page(offset);
 464                 page_length = remain;
 465                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 466                         page_length = PAGE_SIZE - shmem_page_offset;
 467
 468                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 469                         (page_to_phys(page) & (1 << 17)) != 0;
 470
 471                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 472                                        user_data, page_do_bit17_swizzling,
 473                                        needs_clflush);
 474                 if (ret == 0)
 475                         goto next_page;
 476
 477                 mutex_unlock(&dev->struct_mutex);
 478
 479                 if (likely(!i915_prefault_disable) && !prefaulted) {
 480                         ret = fault_in_multipages_writeable(user_data, remain);
 481                         /* Userspace is tricking us, but we've already clobbered
 482                          * its pages with the prefault and promised to write the
 483                          * data up to the first fault. Hence ignore any errors
 484                          * and just continue. */
 485                         (void)ret;
 486                         prefaulted = 1;
 487                 }
 488
 489                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 490                                        user_data, page_do_bit17_swizzling,
 491                                        needs_clflush);
 492
 493                 mutex_lock(&dev->struct_mutex);
 494
 495 next_page:
 496                 mark_page_accessed(page);
 497
 498                 if (ret)
 499                         goto out;
 500
 501                 remain -= page_length;
 502                 user_data += page_length;
 503                 offset += page_length;
 504         }
 505
 506 out:
 507         i915_gem_object_unpin_pages(obj);
 508
 509         return ret;
 510 }
 511
 512 /**
 513  * Reads data from the object referenced by handle.
 514  *
 515  * On error, the contents of *data are undefined.
 516  */
 517 int
 518 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 519                      struct drm_file *file)
 520 {
 521         struct drm_i915_gem_pread *args = data;
 522         struct drm_i915_gem_object *obj;
 523         int ret = 0;
 524
 525         if (args->size == 0)
 526                 return 0;
 527
 528         if (!access_ok(VERIFY_WRITE,
 529                        to_user_ptr(args->data_ptr),
 530                        args->size))
 531                 return -EFAULT;
 532
 533         ret = i915_mutex_lock_interruptible(dev);
 534         if (ret)
 535                 return ret;
 536
 537         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 538         if (&obj->base == NULL) {
 539                 ret = -ENOENT;
 540                 goto unlock;
 541         }
 542
 543         /* Bounds check source.  */
 544         if (args->offset > obj->base.size ||
 545             args->size > obj->base.size - args->offset) {
 546                 ret = -EINVAL;
 547                 goto out;
 548         }
 549
 550         /* prime objects have no backing filp to GEM pread/pwrite
 551          * pages from.
 552          */
 553         if (!obj->base.filp) {
 554                 ret = -EINVAL;
 555                 goto out;
 556         }
 557
 558         trace_i915_gem_object_pread(obj, args->offset, args->size);
 559
 560         ret = i915_gem_shmem_pread(dev, obj, args, file);
 561
 562 out:
 563         drm_gem_object_unreference(&obj->base);
 564 unlock:
 565         mutex_unlock(&dev->struct_mutex);
 566         return ret;
 567 }
 568
 569 /* This is the fast write path which cannot handle
 570  * page faults in the source data
 571  */
 572
 573 static inline int
 574 fast_user_write(struct io_mapping *mapping,
 575                 loff_t page_base, int page_offset,
 576                 char __user *user_data,
 577                 int length)
 578 {
 579         void __iomem *vaddr_atomic;
 580         void *vaddr;
 581         unsigned long unwritten;
 582
 583         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 584         /* We can use the cpu mem copy function because this is X86. */
 585         vaddr = (void __force*)vaddr_atomic + page_offset;
 586         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 587                                                       user_data, length);
 588         io_mapping_unmap_atomic(vaddr_atomic);
 589         return unwritten;
 590 }
 591
 592 /**
 593  * This is the fast pwrite path, where we copy the data directly from the
 594  * user into the GTT, uncached.
 595  */
 596 static int
 597 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 598                          struct drm_i915_gem_object *obj,
 599                          struct drm_i915_gem_pwrite *args,
 600                          struct drm_file *file)
 601 {
 602         drm_i915_private_t *dev_priv = dev->dev_private;
 603         ssize_t remain;
 604         loff_t offset, page_base;
 605         char __user *user_data;
 606         int page_offset, page_length, ret;
 607
 608         ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
 609         if (ret)
 610                 goto out;
 611
 612         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 613         if (ret)
 614                 goto out_unpin;
 615
 616         ret = i915_gem_object_put_fence(obj);
 617         if (ret)
 618                 goto out_unpin;
 619
 620         user_data = to_user_ptr(args->data_ptr);
 621         remain = args->size;
 622
 623         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 624
 625         while (remain > 0) {
 626                 /* Operation in this page
 627                  *
 628                  * page_base = page offset within aperture
 629                  * page_offset = offset within page
 630                  * page_length = bytes to copy for this page
 631                  */
 632                 page_base = offset & PAGE_MASK;
 633                 page_offset = offset_in_page(offset);
 634                 page_length = remain;
 635                 if ((page_offset + remain) > PAGE_SIZE)
 636                         page_length = PAGE_SIZE - page_offset;
 637
 638                 /* If we get a fault while copying data, then (presumably) our
 639                  * source page isn't available.  Return the error and we'll
 640                  * retry in the slow path.
 641                  */
 642                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
 643                                     page_offset, user_data, page_length)) {
 644                         ret = -EFAULT;
 645                         goto out_unpin;
 646                 }
 647
 648                 remain -= page_length;
 649                 user_data += page_length;
 650                 offset += page_length;
 651         }
 652
 653 out_unpin:
 654         i915_gem_object_unpin(obj);
 655 out:
 656         return ret;
 657 }
 658
 659 /* Per-page copy function for the shmem pwrite fastpath.
 660  * Flushes invalid cachelines before writing to the target if
 661  * needs_clflush_before is set and flushes out any written cachelines after
 662  * writing if needs_clflush is set. */
 663 static int
 664 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 665                   char __user *user_data,
 666                   bool page_do_bit17_swizzling,
 667                   bool needs_clflush_before,
 668                   bool needs_clflush_after)
 669 {
 670         char *vaddr;
 671         int ret;
 672
 673         if (unlikely(page_do_bit17_swizzling))
 674                 return -EINVAL;
 675
 676         vaddr = kmap_atomic(page);
 677         if (needs_clflush_before)
 678                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 679                                        page_length);
 680         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 681                                                 user_data,
 682                                                 page_length);
 683         if (needs_clflush_after)
 684                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 685                                        page_length);
 686         kunmap_atomic(vaddr);
 687
 688         return ret ? -EFAULT : 0;
 689 }
 690
 691 /* Only difference to the fast-path function is that this can handle bit17
 692  * and uses non-atomic copy and kmap functions. */
 693 static int
 694 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 695                   char __user *user_data,
 696                   bool page_do_bit17_swizzling,
 697                   bool needs_clflush_before,
 698                   bool needs_clflush_after)
 699 {
 700         char *vaddr;
 701         int ret;
 702
 703         vaddr = kmap(page);
 704         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 705                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 706                                              page_length,
 707                                              page_do_bit17_swizzling);
 708         if (page_do_bit17_swizzling)
 709                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 710                                                 user_data,
 711                                                 page_length);
 712         else
 713                 ret = __copy_from_user(vaddr + shmem_page_offset,
 714                                        user_data,
 715                                        page_length);
 716         if (needs_clflush_after)
 717                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 718                                              page_length,
 719                                              page_do_bit17_swizzling);
 720         kunmap(page);
 721
 722         return ret ? -EFAULT : 0;
 723 }
 724
 725 static int
 726 i915_gem_shmem_pwrite(struct drm_device *dev,
 727                       struct drm_i915_gem_object *obj,
 728                       struct drm_i915_gem_pwrite *args,
 729                       struct drm_file *file)
 730 {
 731         ssize_t remain;
 732         loff_t offset;
 733         char __user *user_data;
 734         int shmem_page_offset, page_length, ret = 0;
 735         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 736         int hit_slowpath = 0;
 737         int needs_clflush_after = 0;
 738         int needs_clflush_before = 0;
 739         struct sg_page_iter sg_iter;
 740
 741         user_data = to_user_ptr(args->data_ptr);
 742         remain = args->size;
 743
 744         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 745
 746         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 747                 /* If we're not in the cpu write domain, set ourself into the gtt
 748                  * write domain and manually flush cachelines (if required). This
 749                  * optimizes for the case when the gpu will use the data
 750                  * right away and we therefore have to clflush anyway. */
 751                 needs_clflush_after = cpu_write_needs_clflush(obj);
 752                 ret = i915_gem_object_wait_rendering(obj, false);
 753                 if (ret)
 754                         return ret;
 755         }
 756         /* Same trick applies to invalidate partially written cachelines read
 757          * before writing. */
 758         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
 759                 needs_clflush_before =
 760                         !cpu_cache_is_coherent(dev, obj->cache_level);
 761
 762         ret = i915_gem_object_get_pages(obj);
 763         if (ret)
 764                 return ret;
 765
 766         i915_gem_object_pin_pages(obj);
 767
 768         offset = args->offset;
 769         obj->dirty = 1;
 770
 771         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 772                          offset >> PAGE_SHIFT) {
 773                 struct page *page = sg_page_iter_page(&sg_iter);
 774                 int partial_cacheline_write;
 775
 776                 if (remain <= 0)
 777                         break;
 778
 779                 /* Operation in this page
 780                  *
 781                  * shmem_page_offset = offset within page in shmem file
 782                  * page_length = bytes to copy for this page
 783                  */
 784                 shmem_page_offset = offset_in_page(offset);
 785
 786                 page_length = remain;
 787                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 788                         page_length = PAGE_SIZE - shmem_page_offset;
 789
 790                 /* If we don't overwrite a cacheline completely we need to be
 791                  * careful to have up-to-date data by first clflushing. Don't
 792                  * overcomplicate things and flush the entire patch. */
 793                 partial_cacheline_write = needs_clflush_before &&
 794                         ((shmem_page_offset | page_length)
 795                                 & (boot_cpu_data.x86_clflush_size - 1));
 796
 797                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 798                         (page_to_phys(page) & (1 << 17)) != 0;
 799
 800                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 801                                         user_data, page_do_bit17_swizzling,
 802                                         partial_cacheline_write,
 803                                         needs_clflush_after);
 804                 if (ret == 0)
 805                         goto next_page;
 806
 807                 hit_slowpath = 1;
 808                 mutex_unlock(&dev->struct_mutex);
 809                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 810                                         user_data, page_do_bit17_swizzling,
 811                                         partial_cacheline_write,
 812                                         needs_clflush_after);
 813
 814                 mutex_lock(&dev->struct_mutex);
 815
 816 next_page:
 817                 set_page_dirty(page);
 818                 mark_page_accessed(page);
 819
 820                 if (ret)
 821                         goto out;
 822
 823                 remain -= page_length;
 824                 user_data += page_length;
 825                 offset += page_length;
 826         }
 827
 828 out:
 829         i915_gem_object_unpin_pages(obj);
 830
 831         if (hit_slowpath) {
 832                 /*
 833                  * Fixup: Flush cpu caches in case we didn't flush the dirty
 834                  * cachelines in-line while writing and the object moved
 835                  * out of the cpu write domain while we've dropped the lock.
 836                  */
 837                 if (!needs_clflush_after &&
 838                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 839                         if (i915_gem_clflush_object(obj, obj->pin_display))
 840                                 i915_gem_chipset_flush(dev);
 841                 }
 842         }
 843
 844         if (needs_clflush_after)
 845                 i915_gem_chipset_flush(dev);
 846
 847         return ret;
 848 }
 849
 850 /**
 851  * Writes data to the object referenced by handle.
 852  *
 853  * On error, the contents of the buffer that were to be modified are undefined.
 854  */
 855 int
 856 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 857                       struct drm_file *file)
 858 {
 859         struct drm_i915_gem_pwrite *args = data;
 860         struct drm_i915_gem_object *obj;
 861         int ret;
 862
 863         if (args->size == 0)
 864                 return 0;
 865
 866         if (!access_ok(VERIFY_READ,
 867                        to_user_ptr(args->data_ptr),
 868                        args->size))
 869                 return -EFAULT;
 870
 871         if (likely(!i915_prefault_disable)) {
 872                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
 873                                                    args->size);
 874                 if (ret)
 875                         return -EFAULT;
 876         }
 877
 878         ret = i915_mutex_lock_interruptible(dev);
 879         if (ret)
 880                 return ret;
 881
 882         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 883         if (&obj->base == NULL) {
 884                 ret = -ENOENT;
 885                 goto unlock;
 886         }
 887
 888         /* Bounds check destination. */
 889         if (args->offset > obj->base.size ||
 890             args->size > obj->base.size - args->offset) {
 891                 ret = -EINVAL;
 892                 goto out;
 893         }
 894
 895         /* prime objects have no backing filp to GEM pread/pwrite
 896          * pages from.
 897          */
 898         if (!obj->base.filp) {
 899                 ret = -EINVAL;
 900                 goto out;
 901         }
 902
 903         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 904
 905         ret = -EFAULT;
 906         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 907          * it would end up going through the fenced access, and we'll get
 908          * different detiling behavior between reading and writing.
 909          * pread/pwrite currently are reading and writing from the CPU
 910          * perspective, requiring manual detiling by the client.
 911          */
 912         if (obj->phys_obj) {
 913                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 914                 goto out;
 915         }
 916
 917         if (obj->tiling_mode == I915_TILING_NONE &&
 918             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
 919             cpu_write_needs_clflush(obj)) {
 920                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 921                 /* Note that the gtt paths might fail with non-page-backed user
 922                  * pointers (e.g. gtt mappings when moving data between
 923                  * textures). Fallback to the shmem path in that case. */
 924         }
 925
 926         if (ret == -EFAULT || ret == -ENOSPC)
 927                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 928
 929 out:
 930         drm_gem_object_unreference(&obj->base);
 931 unlock:
 932         mutex_unlock(&dev->struct_mutex);
 933         return ret;
 934 }
 935
 936 int
 937 i915_gem_check_wedge(struct i915_gpu_error *error,
 938                      bool interruptible)
 939 {
 940         if (i915_reset_in_progress(error)) {
 941                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 942                  * -EIO unconditionally for these. */
 943                 if (!interruptible)
 944                         return -EIO;
 945
 946                 /* Recovery complete, but the reset failed ... */
 947                 if (i915_terminally_wedged(error))
 948                         return -EIO;
 949
 950                 return -EAGAIN;
 951         }
 952
 953         return 0;
 954 }
 955
 956 /*
 957  * Compare seqno against outstanding lazy request. Emit a request if they are
 958  * equal.
 959  */
 960 static int
 961 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
 962 {
 963         int ret;
 964
 965         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 966
 967         ret = 0;
 968         if (seqno == ring->outstanding_lazy_seqno)
 969                 ret = i915_add_request(ring, NULL);
 970
 971         return ret;
 972 }
 973
 974 static void fake_irq(unsigned long data)
 975 {
 976         wake_up_process((struct task_struct *)data);
 977 }
 978
 979 static bool missed_irq(struct drm_i915_private *dev_priv,
 980                        struct intel_ring_buffer *ring)
 981 {
 982         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
 983 }
 984
 985 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
 986 {
 987         if (file_priv == NULL)
 988                 return true;
 989
 990         return !atomic_xchg(&file_priv->rps_wait_boost, true);
 991 }
 992
 993 /**
 994  * __wait_seqno - wait until execution of seqno has finished
 995  * @ring: the ring expected to report seqno
 996  * @seqno: duh!
 997  * @reset_counter: reset sequence associated with the given seqno
 998  * @interruptible: do an interruptible wait (normally yes)
 999  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1000  *
1001  * Note: It is of utmost importance that the passed in seqno and reset_counter
1002  * values have been read by the caller in an smp safe manner. Where read-side
1003  * locks are involved, it is sufficient to read the reset_counter before
1004  * unlocking the lock that protects the seqno. For lockless tricks, the
1005  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1006  * inserted.
1007  *
1008  * Returns 0 if the seqno was found within the alloted time. Else returns the
1009  * errno with remaining time filled in timeout argument.
1010  */
1011 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1012                         unsigned reset_counter,
1013                         bool interruptible,
1014                         struct timespec *timeout,
1015                         struct drm_i915_file_private *file_priv)
1016 {
1017         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1018         struct timespec before, now;
1019         DEFINE_WAIT(wait);
1020         long timeout_jiffies;
1021         int ret;
1022
1023         WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
1024
1025         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1026                 return 0;
1027
1028         timeout_jiffies = timeout ? timespec_to_jiffies_timeout(timeout) : 1;
1029
1030         if (dev_priv->info->gen >= 6 && can_wait_boost(file_priv)) {
1031                 gen6_rps_boost(dev_priv);
1032                 if (file_priv)
1033                         mod_delayed_work(dev_priv->wq,
1034                                          &file_priv->mm.idle_work,
1035                                          msecs_to_jiffies(100));
1036         }
1037
1038         if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)) &&
1039             WARN_ON(!ring->irq_get(ring)))
1040                 return -ENODEV;
1041
1042         /* Record current time in case interrupted by signal, or wedged */
1043         trace_i915_gem_request_wait_begin(ring, seqno);
1044         getrawmonotonic(&before);
1045         for (;;) {
1046                 struct timer_list timer;
1047                 unsigned long expire;
1048
1049                 prepare_to_wait(&ring->irq_queue, &wait,
1050                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1051
1052                 /* We need to check whether any gpu reset happened in between
1053                  * the caller grabbing the seqno and now ... */
1054                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1055                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1056                          * is truely gone. */
1057                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1058                         if (ret == 0)
1059                                 ret = -EAGAIN;
1060                         break;
1061                 }
1062
1063                 if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
1064                         ret = 0;
1065                         break;
1066                 }
1067
1068                 if (interruptible && signal_pending(current)) {
1069                         ret = -ERESTARTSYS;
1070                         break;
1071                 }
1072
1073                 if (timeout_jiffies <= 0) {
1074                         ret = -ETIME;
1075                         break;
1076                 }
1077
1078                 timer.function = NULL;
1079                 if (timeout || missed_irq(dev_priv, ring)) {
1080                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1081                         expire = jiffies + (missed_irq(dev_priv, ring) ? 1: timeout_jiffies);
1082                         mod_timer(&timer, expire);
1083                 }
1084
1085                 io_schedule();
1086
1087                 if (timeout)
1088                         timeout_jiffies = expire - jiffies;
1089
1090                 if (timer.function) {
1091                         del_singleshot_timer_sync(&timer);
1092                         destroy_timer_on_stack(&timer);
1093                 }
1094         }
1095         getrawmonotonic(&now);
1096         trace_i915_gem_request_wait_end(ring, seqno);
1097
1098         ring->irq_put(ring);
1099
1100         finish_wait(&ring->irq_queue, &wait);
1101
1102         if (timeout) {
1103                 struct timespec sleep_time = timespec_sub(now, before);
1104                 *timeout = timespec_sub(*timeout, sleep_time);
1105                 if (!timespec_valid(timeout)) /* i.e. negative time remains */
1106                         set_normalized_timespec(timeout, 0, 0);
1107         }
1108
1109         return ret;
1110 }
1111
1112 /**
1113  * Waits for a sequence number to be signaled, and cleans up the
1114  * request and object lists appropriately for that event.
1115  */
1116 int
1117 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1118 {
1119         struct drm_device *dev = ring->dev;
1120         struct drm_i915_private *dev_priv = dev->dev_private;
1121         bool interruptible = dev_priv->mm.interruptible;
1122         int ret;
1123
1124         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1125         BUG_ON(seqno == 0);
1126
1127         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1128         if (ret)
1129                 return ret;
1130
1131         ret = i915_gem_check_olr(ring, seqno);
1132         if (ret)
1133                 return ret;
1134
1135         return __wait_seqno(ring, seqno,
1136                             atomic_read(&dev_priv->gpu_error.reset_counter),
1137                             interruptible, NULL, NULL);
1138 }
1139
1140 static int
1141 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
1142                                      struct intel_ring_buffer *ring)
1143 {
1144         i915_gem_retire_requests_ring(ring);
1145
1146         /* Manually manage the write flush as we may have not yet
1147          * retired the buffer.
1148          *
1149          * Note that the last_write_seqno is always the earlier of
1150          * the two (read/write) seqno, so if we haved successfully waited,
1151          * we know we have passed the last write.
1152          */
1153         obj->last_write_seqno = 0;
1154         obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1155
1156         return 0;
1157 }
1158
1159 /**
1160  * Ensures that all rendering to the object has completed and the object is
1161  * safe to unbind from the GTT or access from the CPU.
1162  */
1163 static __must_check int
1164 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1165                                bool readonly)
1166 {
1167         struct intel_ring_buffer *ring = obj->ring;
1168         u32 seqno;
1169         int ret;
1170
1171         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1172         if (seqno == 0)
1173                 return 0;
1174
1175         ret = i915_wait_seqno(ring, seqno);
1176         if (ret)
1177                 return ret;
1178
1179         return i915_gem_object_wait_rendering__tail(obj, ring);
1180 }
1181
1182 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1183  * as the object state may change during this call.
1184  */
1185 static __must_check int
1186 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1187                                             struct drm_file *file,
1188                                             bool readonly)
1189 {
1190         struct drm_device *dev = obj->base.dev;
1191         struct drm_i915_private *dev_priv = dev->dev_private;
1192         struct intel_ring_buffer *ring = obj->ring;
1193         unsigned reset_counter;
1194         u32 seqno;
1195         int ret;
1196
1197         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1198         BUG_ON(!dev_priv->mm.interruptible);
1199
1200         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1201         if (seqno == 0)
1202                 return 0;
1203
1204         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1205         if (ret)
1206                 return ret;
1207
1208         ret = i915_gem_check_olr(ring, seqno);
1209         if (ret)
1210                 return ret;
1211
1212         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1213         mutex_unlock(&dev->struct_mutex);
1214         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file->driver_priv);
1215         mutex_lock(&dev->struct_mutex);
1216         if (ret)
1217                 return ret;
1218
1219         return i915_gem_object_wait_rendering__tail(obj, ring);
1220 }
1221
1222 /**
1223  * Called when user space prepares to use an object with the CPU, either
1224  * through the mmap ioctl's mapping or a GTT mapping.
1225  */
1226 int
1227 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1228                           struct drm_file *file)
1229 {
1230         struct drm_i915_gem_set_domain *args = data;
1231         struct drm_i915_gem_object *obj;
1232         uint32_t read_domains = args->read_domains;
1233         uint32_t write_domain = args->write_domain;
1234         int ret;
1235
1236         /* Only handle setting domains to types used by the CPU. */
1237         if (write_domain & I915_GEM_GPU_DOMAINS)
1238                 return -EINVAL;
1239
1240         if (read_domains & I915_GEM_GPU_DOMAINS)
1241                 return -EINVAL;
1242
1243         /* Having something in the write domain implies it's in the read
1244          * domain, and only that read domain.  Enforce that in the request.
1245          */
1246         if (write_domain != 0 && read_domains != write_domain)
1247                 return -EINVAL;
1248
1249         ret = i915_mutex_lock_interruptible(dev);
1250         if (ret)
1251                 return ret;
1252
1253         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1254         if (&obj->base == NULL) {
1255                 ret = -ENOENT;
1256                 goto unlock;
1257         }
1258
1259         /* Try to flush the object off the GPU without holding the lock.
1260          * We will repeat the flush holding the lock in the normal manner
1261          * to catch cases where we are gazumped.
1262          */
1263         ret = i915_gem_object_wait_rendering__nonblocking(obj, file, !write_domain);
1264         if (ret)
1265                 goto unref;
1266
1267         if (read_domains & I915_GEM_DOMAIN_GTT) {
1268                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1269
1270                 /* Silently promote "you're not bound, there was nothing to do"
1271                  * to success, since the client was just asking us to
1272                  * make sure everything was done.
1273                  */
1274                 if (ret == -EINVAL)
1275                         ret = 0;
1276         } else {
1277                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1278         }
1279
1280 unref:
1281         drm_gem_object_unreference(&obj->base);
1282 unlock:
1283         mutex_unlock(&dev->struct_mutex);
1284         return ret;
1285 }
1286
1287 /**
1288  * Called when user space has done writes to this buffer
1289  */
1290 int
1291 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1292                          struct drm_file *file)
1293 {
1294         struct drm_i915_gem_sw_finish *args = data;
1295         struct drm_i915_gem_object *obj;
1296         int ret = 0;
1297
1298         ret = i915_mutex_lock_interruptible(dev);
1299         if (ret)
1300                 return ret;
1301
1302         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1303         if (&obj->base == NULL) {
1304                 ret = -ENOENT;
1305                 goto unlock;
1306         }
1307
1308         /* Pinned buffers may be scanout, so flush the cache */
1309         if (obj->pin_display)
1310                 i915_gem_object_flush_cpu_write_domain(obj, true);
1311
1312         drm_gem_object_unreference(&obj->base);
1313 unlock:
1314         mutex_unlock(&dev->struct_mutex);
1315         return ret;
1316 }
1317
1318 /**
1319  * Maps the contents of an object, returning the address it is mapped
1320  * into.
1321  *
1322  * While the mapping holds a reference on the contents of the object, it doesn't
1323  * imply a ref on the object itself.
1324  */
1325 int
1326 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1327                     struct drm_file *file)
1328 {
1329         struct drm_i915_gem_mmap *args = data;
1330         struct drm_gem_object *obj;
1331         unsigned long addr;
1332
1333         obj = drm_gem_object_lookup(dev, file, args->handle);
1334         if (obj == NULL)
1335                 return -ENOENT;
1336
1337         /* prime objects have no backing filp to GEM mmap
1338          * pages from.
1339          */
1340         if (!obj->filp) {
1341                 drm_gem_object_unreference_unlocked(obj);
1342                 return -EINVAL;
1343         }
1344
1345         addr = vm_mmap(obj->filp, 0, args->size,
1346                        PROT_READ | PROT_WRITE, MAP_SHARED,
1347                        args->offset);
1348         drm_gem_object_unreference_unlocked(obj);
1349         if (IS_ERR((void *)addr))
1350                 return addr;
1351
1352         args->addr_ptr = (uint64_t) addr;
1353
1354         return 0;
1355 }
1356
1357 /**
1358  * i915_gem_fault - fault a page into the GTT
1359  * vma: VMA in question
1360  * vmf: fault info
1361  *
1362  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1363  * from userspace.  The fault handler takes care of binding the object to
1364  * the GTT (if needed), allocating and programming a fence register (again,
1365  * only if needed based on whether the old reg is still valid or the object
1366  * is tiled) and inserting a new PTE into the faulting process.
1367  *
1368  * Note that the faulting process may involve evicting existing objects
1369  * from the GTT and/or fence registers to make room.  So performance may
1370  * suffer if the GTT working set is large or there are few fence registers
1371  * left.
1372  */
1373 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1374 {
1375         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1376         struct drm_device *dev = obj->base.dev;
1377         drm_i915_private_t *dev_priv = dev->dev_private;
1378         pgoff_t page_offset;
1379         unsigned long pfn;
1380         int ret = 0;
1381         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1382
1383         /* We don't use vmf->pgoff since that has the fake offset */
1384         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1385                 PAGE_SHIFT;
1386
1387         ret = i915_mutex_lock_interruptible(dev);
1388         if (ret)
1389                 goto out;
1390
1391         trace_i915_gem_object_fault(obj, page_offset, true, write);
1392
1393         /* Access to snoopable pages through the GTT is incoherent. */
1394         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1395                 ret = -EINVAL;
1396                 goto unlock;
1397         }
1398
1399         /* Now bind it into the GTT if needed */
1400         ret = i915_gem_obj_ggtt_pin(obj,  0, true, false);
1401         if (ret)
1402                 goto unlock;
1403
1404         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1405         if (ret)
1406                 goto unpin;
1407
1408         ret = i915_gem_object_get_fence(obj);
1409         if (ret)
1410                 goto unpin;
1411
1412         obj->fault_mappable = true;
1413
1414         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1415         pfn >>= PAGE_SHIFT;
1416         pfn += page_offset;
1417
1418         /* Finally, remap it using the new GTT offset */
1419         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1420 unpin:
1421         i915_gem_object_unpin(obj);
1422 unlock:
1423         mutex_unlock(&dev->struct_mutex);
1424 out:
1425         switch (ret) {
1426         case -EIO:
1427                 /* If this -EIO is due to a gpu hang, give the reset code a
1428                  * chance to clean up the mess. Otherwise return the proper
1429                  * SIGBUS. */
1430                 if (i915_terminally_wedged(&dev_priv->gpu_error))
1431                         return VM_FAULT_SIGBUS;
1432         case -EAGAIN:
1433                 /*
1434                  * EAGAIN means the gpu is hung and we'll wait for the error
1435                  * handler to reset everything when re-faulting in
1436                  * i915_mutex_lock_interruptible.
1437                  */
1438         case 0:
1439         case -ERESTARTSYS:
1440         case -EINTR:
1441         case -EBUSY:
1442                 /*
1443                  * EBUSY is ok: this just means that another thread
1444                  * already did the job.
1445                  */
1446                 return VM_FAULT_NOPAGE;
1447         case -ENOMEM:
1448                 return VM_FAULT_OOM;
1449         case -ENOSPC:
1450                 return VM_FAULT_SIGBUS;
1451         default:
1452                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1453                 return VM_FAULT_SIGBUS;
1454         }
1455 }
1456
1457 /**
1458  * i915_gem_release_mmap - remove physical page mappings
1459  * @obj: obj in question
1460  *
1461  * Preserve the reservation of the mmapping with the DRM core code, but
1462  * relinquish ownership of the pages back to the system.
1463  *
1464  * It is vital that we remove the page mapping if we have mapped a tiled
1465  * object through the GTT and then lose the fence register due to
1466  * resource pressure. Similarly if the object has been moved out of the
1467  * aperture, than pages mapped into userspace must be revoked. Removing the
1468  * mapping will then trigger a page fault on the next user access, allowing
1469  * fixup by i915_gem_fault().
1470  */
1471 void
1472 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1473 {
1474         if (!obj->fault_mappable)
1475                 return;
1476
1477         drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
1478         obj->fault_mappable = false;
1479 }
1480
1481 uint32_t
1482 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1483 {
1484         uint32_t gtt_size;
1485
1486         if (INTEL_INFO(dev)->gen >= 4 ||
1487             tiling_mode == I915_TILING_NONE)
1488                 return size;
1489
1490         /* Previous chips need a power-of-two fence region when tiling */
1491         if (INTEL_INFO(dev)->gen == 3)
1492                 gtt_size = 1024*1024;
1493         else
1494                 gtt_size = 512*1024;
1495
1496         while (gtt_size < size)
1497                 gtt_size <<= 1;
1498
1499         return gtt_size;
1500 }
1501
1502 /**
1503  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1504  * @obj: object to check
1505  *
1506  * Return the required GTT alignment for an object, taking into account
1507  * potential fence register mapping.
1508  */
1509 uint32_t
1510 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1511                            int tiling_mode, bool fenced)
1512 {
1513         /*
1514          * Minimum alignment is 4k (GTT page size), but might be greater
1515          * if a fence register is needed for the object.
1516          */
1517         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1518             tiling_mode == I915_TILING_NONE)
1519                 return 4096;
1520
1521         /*
1522          * Previous chips need to be aligned to the size of the smallest
1523          * fence register that can contain the object.
1524          */
1525         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1526 }
1527
1528 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1529 {
1530         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1531         int ret;
1532
1533         if (drm_vma_node_has_offset(&obj->base.vma_node))
1534                 return 0;
1535
1536         dev_priv->mm.shrinker_no_lock_stealing = true;
1537
1538         ret = drm_gem_create_mmap_offset(&obj->base);
1539         if (ret != -ENOSPC)
1540                 goto out;
1541
1542         /* Badly fragmented mmap space? The only way we can recover
1543          * space is by destroying unwanted objects. We can't randomly release
1544          * mmap_offsets as userspace expects them to be persistent for the
1545          * lifetime of the objects. The closest we can is to release the
1546          * offsets on purgeable objects by truncating it and marking it purged,
1547          * which prevents userspace from ever using that object again.
1548          */
1549         i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1550         ret = drm_gem_create_mmap_offset(&obj->base);
1551         if (ret != -ENOSPC)
1552                 goto out;
1553
1554         i915_gem_shrink_all(dev_priv);
1555         ret = drm_gem_create_mmap_offset(&obj->base);
1556 out:
1557         dev_priv->mm.shrinker_no_lock_stealing = false;
1558
1559         return ret;
1560 }
1561
1562 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1563 {
1564         drm_gem_free_mmap_offset(&obj->base);
1565 }
1566
1567 int
1568 i915_gem_mmap_gtt(struct drm_file *file,
1569                   struct drm_device *dev,
1570                   uint32_t handle,
1571                   uint64_t *offset)
1572 {
1573         struct drm_i915_private *dev_priv = dev->dev_private;
1574         struct drm_i915_gem_object *obj;
1575         int ret;
1576
1577         ret = i915_mutex_lock_interruptible(dev);
1578         if (ret)
1579                 return ret;
1580
1581         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1582         if (&obj->base == NULL) {
1583                 ret = -ENOENT;
1584                 goto unlock;
1585         }
1586
1587         if (obj->base.size > dev_priv->gtt.mappable_end) {
1588                 ret = -E2BIG;
1589                 goto out;
1590         }
1591
1592         if (obj->madv != I915_MADV_WILLNEED) {
1593                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1594                 ret = -EINVAL;
1595                 goto out;
1596         }
1597
1598         ret = i915_gem_object_create_mmap_offset(obj);
1599         if (ret)
1600                 goto out;
1601
1602         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1603
1604 out:
1605         drm_gem_object_unreference(&obj->base);
1606 unlock:
1607         mutex_unlock(&dev->struct_mutex);
1608         return ret;
1609 }
1610
1611 /**
1612  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1613  * @dev: DRM device
1614  * @data: GTT mapping ioctl data
1615  * @file: GEM object info
1616  *
1617  * Simply returns the fake offset to userspace so it can mmap it.
1618  * The mmap call will end up in drm_gem_mmap(), which will set things
1619  * up so we can get faults in the handler above.
1620  *
1621  * The fault handler will take care of binding the object into the GTT
1622  * (since it may have been evicted to make room for something), allocating
1623  * a fence register, and mapping the appropriate aperture address into
1624  * userspace.
1625  */
1626 int
1627 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1628                         struct drm_file *file)
1629 {
1630         struct drm_i915_gem_mmap_gtt *args = data;
1631
1632         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1633 }
1634
1635 /* Immediately discard the backing storage */
1636 static void
1637 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1638 {
1639         struct inode *inode;
1640
1641         i915_gem_object_free_mmap_offset(obj);
1642
1643         if (obj->base.filp == NULL)
1644                 return;
1645
1646         /* Our goal here is to return as much of the memory as
1647          * is possible back to the system as we are called from OOM.
1648          * To do this we must instruct the shmfs to drop all of its
1649          * backing pages, *now*.
1650          */
1651         inode = file_inode(obj->base.filp);
1652         shmem_truncate_range(inode, 0, (loff_t)-1);
1653
1654         obj->madv = __I915_MADV_PURGED;
1655 }
1656
1657 static inline int
1658 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1659 {
1660         return obj->madv == I915_MADV_DONTNEED;
1661 }
1662
1663 static void
1664 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1665 {
1666         struct sg_page_iter sg_iter;
1667         int ret;
1668
1669         BUG_ON(obj->madv == __I915_MADV_PURGED);
1670
1671         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1672         if (ret) {
1673                 /* In the event of a disaster, abandon all caches and
1674                  * hope for the best.
1675                  */
1676                 WARN_ON(ret != -EIO);
1677                 i915_gem_clflush_object(obj, true);
1678                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1679         }
1680
1681         if (i915_gem_object_needs_bit17_swizzle(obj))
1682                 i915_gem_object_save_bit_17_swizzle(obj);
1683
1684         if (obj->madv == I915_MADV_DONTNEED)
1685                 obj->dirty = 0;
1686
1687         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1688                 struct page *page = sg_page_iter_page(&sg_iter);
1689
1690                 if (obj->dirty)
1691                         set_page_dirty(page);
1692
1693                 if (obj->madv == I915_MADV_WILLNEED)
1694                         mark_page_accessed(page);
1695
1696                 page_cache_release(page);
1697         }
1698         obj->dirty = 0;
1699
1700         sg_free_table(obj->pages);
1701         kfree(obj->pages);
1702 }
1703
1704 int
1705 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1706 {
1707         const struct drm_i915_gem_object_ops *ops = obj->ops;
1708
1709         if (obj->pages == NULL)
1710                 return 0;
1711
1712         if (obj->pages_pin_count)
1713                 return -EBUSY;
1714
1715         BUG_ON(i915_gem_obj_bound_any(obj));
1716
1717         /* ->put_pages might need to allocate memory for the bit17 swizzle
1718          * array, hence protect them from being reaped by removing them from gtt
1719          * lists early. */
1720         list_del(&obj->global_list);
1721
1722         ops->put_pages(obj);
1723         obj->pages = NULL;
1724
1725         if (i915_gem_object_is_purgeable(obj))
1726                 i915_gem_object_truncate(obj);
1727
1728         return 0;
1729 }
1730
1731 static unsigned long
1732 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
1733                   bool purgeable_only)
1734 {
1735         struct list_head still_bound_list;
1736         struct drm_i915_gem_object *obj, *next;
1737         unsigned long count = 0;
1738
1739         list_for_each_entry_safe(obj, next,
1740                                  &dev_priv->mm.unbound_list,
1741                                  global_list) {
1742                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1743                     i915_gem_object_put_pages(obj) == 0) {
1744                         count += obj->base.size >> PAGE_SHIFT;
1745                         if (count >= target)
1746                                 return count;
1747                 }
1748         }
1749
1750         /*
1751          * As we may completely rewrite the bound list whilst unbinding
1752          * (due to retiring requests) we have to strictly process only
1753          * one element of the list at the time, and recheck the list
1754          * on every iteration.
1755          */
1756         INIT_LIST_HEAD(&still_bound_list);
1757         while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
1758                 struct i915_vma *vma, *v;
1759
1760                 obj = list_first_entry(&dev_priv->mm.bound_list,
1761                                        typeof(*obj), global_list);
1762                 list_move_tail(&obj->global_list, &still_bound_list);
1763
1764                 if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
1765                         continue;
1766
1767                 /*
1768                  * Hold a reference whilst we unbind this object, as we may
1769                  * end up waiting for and retiring requests. This might
1770                  * release the final reference (held by the active list)
1771                  * and result in the object being freed from under us.
1772                  * in this object being freed.
1773                  *
1774                  * Note 1: Shrinking the bound list is special since only active
1775                  * (and hence bound objects) can contain such limbo objects, so
1776                  * we don't need special tricks for shrinking the unbound list.
1777                  * The only other place where we have to be careful with active
1778                  * objects suddenly disappearing due to retiring requests is the
1779                  * eviction code.
1780                  *
1781                  * Note 2: Even though the bound list doesn't hold a reference
1782                  * to the object we can safely grab one here: The final object
1783                  * unreferencing and the bound_list are both protected by the
1784                  * dev->struct_mutex and so we won't ever be able to observe an
1785                  * object on the bound_list with a reference count equals 0.
1786                  */
1787                 drm_gem_object_reference(&obj->base);
1788
1789                 list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
1790                         if (i915_vma_unbind(vma))
1791                                 break;
1792
1793                 if (i915_gem_object_put_pages(obj) == 0)
1794                         count += obj->base.size >> PAGE_SHIFT;
1795
1796                 drm_gem_object_unreference(&obj->base);
1797         }
1798         list_splice(&still_bound_list, &dev_priv->mm.bound_list);
1799
1800         return count;
1801 }
1802
1803 static unsigned long
1804 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1805 {
1806         return __i915_gem_shrink(dev_priv, target, true);
1807 }
1808
1809 static unsigned long
1810 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1811 {
1812         struct drm_i915_gem_object *obj, *next;
1813         long freed = 0;
1814
1815         i915_gem_evict_everything(dev_priv->dev);
1816
1817         list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
1818                                  global_list) {
1819                 if (i915_gem_object_put_pages(obj) == 0)
1820                         freed += obj->base.size >> PAGE_SHIFT;
1821         }
1822         return freed;
1823 }
1824
1825 static int
1826 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1827 {
1828         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1829         int page_count, i;
1830         struct address_space *mapping;
1831         struct sg_table *st;
1832         struct scatterlist *sg;
1833         struct sg_page_iter sg_iter;
1834         struct page *page;
1835         unsigned long last_pfn = 0;     /* suppress gcc warning */
1836         gfp_t gfp;
1837
1838         /* Assert that the object is not currently in any GPU domain. As it
1839          * wasn't in the GTT, there shouldn't be any way it could have been in
1840          * a GPU cache
1841          */
1842         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1843         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1844
1845         st = kmalloc(sizeof(*st), GFP_KERNEL);
1846         if (st == NULL)
1847                 return -ENOMEM;
1848
1849         page_count = obj->base.size / PAGE_SIZE;
1850         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1851                 kfree(st);
1852                 return -ENOMEM;
1853         }
1854
1855         /* Get the list of pages out of our struct file.  They'll be pinned
1856          * at this point until we release them.
1857          *
1858          * Fail silently without starting the shrinker
1859          */
1860         mapping = file_inode(obj->base.filp)->i_mapping;
1861         gfp = mapping_gfp_mask(mapping);
1862         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1863         gfp &= ~(__GFP_IO | __GFP_WAIT);
1864         sg = st->sgl;
1865         st->nents = 0;
1866         for (i = 0; i < page_count; i++) {
1867                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1868                 if (IS_ERR(page)) {
1869                         i915_gem_purge(dev_priv, page_count);
1870                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1871                 }
1872                 if (IS_ERR(page)) {
1873                         /* We've tried hard to allocate the memory by reaping
1874                          * our own buffer, now let the real VM do its job and
1875                          * go down in flames if truly OOM.
1876                          */
1877                         gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
1878                         gfp |= __GFP_IO | __GFP_WAIT;
1879
1880                         i915_gem_shrink_all(dev_priv);
1881                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1882                         if (IS_ERR(page))
1883                                 goto err_pages;
1884
1885                         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1886                         gfp &= ~(__GFP_IO | __GFP_WAIT);
1887                 }
1888 #ifdef CONFIG_SWIOTLB
1889                 if (swiotlb_nr_tbl()) {
1890                         st->nents++;
1891                         sg_set_page(sg, page, PAGE_SIZE, 0);
1892                         sg = sg_next(sg);
1893                         continue;
1894                 }
1895 #endif
1896                 if (!i || page_to_pfn(page) != last_pfn + 1) {
1897                         if (i)
1898                                 sg = sg_next(sg);
1899                         st->nents++;
1900                         sg_set_page(sg, page, PAGE_SIZE, 0);
1901                 } else {
1902                         sg->length += PAGE_SIZE;
1903                 }
1904                 last_pfn = page_to_pfn(page);
1905
1906                 /* Check that the i965g/gm workaround works. */
1907                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
1908         }
1909 #ifdef CONFIG_SWIOTLB
1910         if (!swiotlb_nr_tbl())
1911 #endif
1912                 sg_mark_end(sg);
1913         obj->pages = st;
1914
1915         if (i915_gem_object_needs_bit17_swizzle(obj))
1916                 i915_gem_object_do_bit_17_swizzle(obj);
1917
1918         return 0;
1919
1920 err_pages:
1921         sg_mark_end(sg);
1922         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1923                 page_cache_release(sg_page_iter_page(&sg_iter));
1924         sg_free_table(st);
1925         kfree(st);
1926         return PTR_ERR(page);
1927 }
1928
1929 /* Ensure that the associated pages are gathered from the backing storage
1930  * and pinned into our object. i915_gem_object_get_pages() may be called
1931  * multiple times before they are released by a single call to
1932  * i915_gem_object_put_pages() - once the pages are no longer referenced
1933  * either as a result of memory pressure (reaping pages under the shrinker)
1934  * or as the object is itself released.
1935  */
1936 int
1937 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1938 {
1939         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1940         const struct drm_i915_gem_object_ops *ops = obj->ops;
1941         int ret;
1942
1943         if (obj->pages)
1944                 return 0;
1945
1946         if (obj->madv != I915_MADV_WILLNEED) {
1947                 DRM_ERROR("Attempting to obtain a purgeable object\n");
1948                 return -EINVAL;
1949         }
1950
1951         BUG_ON(obj->pages_pin_count);
1952
1953         ret = ops->get_pages(obj);
1954         if (ret)
1955                 return ret;
1956
1957         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1958         return 0;
1959 }
1960
1961 static void
1962 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1963                                struct intel_ring_buffer *ring)
1964 {
1965         struct drm_device *dev = obj->base.dev;
1966         struct drm_i915_private *dev_priv = dev->dev_private;
1967         u32 seqno = intel_ring_get_seqno(ring);
1968
1969         BUG_ON(ring == NULL);
1970         if (obj->ring != ring && obj->last_write_seqno) {
1971                 /* Keep the seqno relative to the current ring */
1972                 obj->last_write_seqno = seqno;
1973         }
1974         obj->ring = ring;
1975
1976         /* Add a reference if we're newly entering the active list. */
1977         if (!obj->active) {
1978                 drm_gem_object_reference(&obj->base);
1979                 obj->active = 1;
1980         }
1981
1982         list_move_tail(&obj->ring_list, &ring->active_list);
1983
1984         obj->last_read_seqno = seqno;
1985
1986         if (obj->fenced_gpu_access) {
1987                 obj->last_fenced_seqno = seqno;
1988
1989                 /* Bump MRU to take account of the delayed flush */
1990                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1991                         struct drm_i915_fence_reg *reg;
1992
1993                         reg = &dev_priv->fence_regs[obj->fence_reg];
1994                         list_move_tail(&reg->lru_list,
1995                                        &dev_priv->mm.fence_list);
1996                 }
1997         }
1998 }
1999
2000 void i915_vma_move_to_active(struct i915_vma *vma,
2001                              struct intel_ring_buffer *ring)
2002 {
2003         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2004         return i915_gem_object_move_to_active(vma->obj, ring);
2005 }
2006
2007 static void
2008 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2009 {
2010         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2011         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
2012         struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2013
2014         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2015         BUG_ON(!obj->active);
2016
2017         list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2018
2019         list_del_init(&obj->ring_list);
2020         obj->ring = NULL;
2021
2022         obj->last_read_seqno = 0;
2023         obj->last_write_seqno = 0;
2024         obj->base.write_domain = 0;
2025
2026         obj->last_fenced_seqno = 0;
2027         obj->fenced_gpu_access = false;
2028
2029         obj->active = 0;
2030         drm_gem_object_unreference(&obj->base);
2031
2032         WARN_ON(i915_verify_lists(dev));
2033 }
2034
2035 static int
2036 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2037 {
2038         struct drm_i915_private *dev_priv = dev->dev_private;
2039         struct intel_ring_buffer *ring;
2040         int ret, i, j;
2041
2042         /* Carefully retire all requests without writing to the rings */
2043         for_each_ring(ring, dev_priv, i) {
2044                 ret = intel_ring_idle(ring);
2045                 if (ret)
2046                         return ret;
2047         }
2048         i915_gem_retire_requests(dev);
2049
2050         /* Finally reset hw state */
2051         for_each_ring(ring, dev_priv, i) {
2052                 intel_ring_init_seqno(ring, seqno);
2053
2054                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
2055                         ring->sync_seqno[j] = 0;
2056         }
2057
2058         return 0;
2059 }
2060
2061 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2062 {
2063         struct drm_i915_private *dev_priv = dev->dev_private;
2064         int ret;
2065
2066         if (seqno == 0)
2067                 return -EINVAL;
2068
2069         /* HWS page needs to be set less than what we
2070          * will inject to ring
2071          */
2072         ret = i915_gem_init_seqno(dev, seqno - 1);
2073         if (ret)
2074                 return ret;
2075
2076         /* Carefully set the last_seqno value so that wrap
2077          * detection still works
2078          */
2079         dev_priv->next_seqno = seqno;
2080         dev_priv->last_seqno = seqno - 1;
2081         if (dev_priv->last_seqno == 0)
2082                 dev_priv->last_seqno--;
2083
2084         return 0;
2085 }
2086
2087 int
2088 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2089 {
2090         struct drm_i915_private *dev_priv = dev->dev_private;
2091
2092         /* reserve 0 for non-seqno */
2093         if (dev_priv->next_seqno == 0) {
2094                 int ret = i915_gem_init_seqno(dev, 0);
2095                 if (ret)
2096                         return ret;
2097
2098                 dev_priv->next_seqno = 1;
2099         }
2100
2101         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2102         return 0;
2103 }
2104
2105 int __i915_add_request(struct intel_ring_buffer *ring,
2106                        struct drm_file *file,
2107                        struct drm_i915_gem_object *obj,
2108                        u32 *out_seqno)
2109 {
2110         drm_i915_private_t *dev_priv = ring->dev->dev_private;
2111         struct drm_i915_gem_request *request;
2112         u32 request_ring_position, request_start;
2113         int was_empty;
2114         int ret;
2115
2116         request_start = intel_ring_get_tail(ring);
2117         /*
2118          * Emit any outstanding flushes - execbuf can fail to emit the flush
2119          * after having emitted the batchbuffer command. Hence we need to fix
2120          * things up similar to emitting the lazy request. The difference here
2121          * is that the flush _must_ happen before the next request, no matter
2122          * what.
2123          */
2124         ret = intel_ring_flush_all_caches(ring);
2125         if (ret)
2126                 return ret;
2127
2128         request = ring->preallocated_lazy_request;
2129         if (WARN_ON(request == NULL))
2130                 return -ENOMEM;
2131
2132         /* Record the position of the start of the request so that
2133          * should we detect the updated seqno part-way through the
2134          * GPU processing the request, we never over-estimate the
2135          * position of the head.
2136          */
2137         request_ring_position = intel_ring_get_tail(ring);
2138
2139         ret = ring->add_request(ring);
2140         if (ret)
2141                 return ret;
2142
2143         request->seqno = intel_ring_get_seqno(ring);
2144         request->ring = ring;
2145         request->head = request_start;
2146         request->tail = request_ring_position;
2147
2148         /* Whilst this request exists, batch_obj will be on the
2149          * active_list, and so will hold the active reference. Only when this
2150          * request is retired will the the batch_obj be moved onto the
2151          * inactive_list and lose its active reference. Hence we do not need
2152          * to explicitly hold another reference here.
2153          */
2154         request->batch_obj = obj;
2155
2156         /* Hold a reference to the current context so that we can inspect
2157          * it later in case a hangcheck error event fires.
2158          */
2159         request->ctx = ring->last_context;
2160         if (request->ctx)
2161                 i915_gem_context_reference(request->ctx);
2162
2163         request->emitted_jiffies = jiffies;
2164         was_empty = list_empty(&ring->request_list);
2165         list_add_tail(&request->list, &ring->request_list);
2166         request->file_priv = NULL;
2167
2168         if (file) {
2169                 struct drm_i915_file_private *file_priv = file->driver_priv;
2170
2171                 spin_lock(&file_priv->mm.lock);
2172                 request->file_priv = file_priv;
2173                 list_add_tail(&request->client_list,
2174                               &file_priv->mm.request_list);
2175                 spin_unlock(&file_priv->mm.lock);
2176         }
2177
2178         trace_i915_gem_request_add(ring, request->seqno);
2179         ring->outstanding_lazy_seqno = 0;
2180         ring->preallocated_lazy_request = NULL;
2181
2182         if (!dev_priv->ums.mm_suspended) {
2183                 i915_queue_hangcheck(ring->dev);
2184
2185                 if (was_empty) {
2186                         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2187                         queue_delayed_work(dev_priv->wq,
2188                                            &dev_priv->mm.retire_work,
2189                                            round_jiffies_up_relative(HZ));
2190                         intel_mark_busy(dev_priv->dev);
2191                 }
2192         }
2193
2194         if (out_seqno)
2195                 *out_seqno = request->seqno;
2196         return 0;
2197 }
2198
2199 static inline void
2200 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2201 {
2202         struct drm_i915_file_private *file_priv = request->file_priv;
2203
2204         if (!file_priv)
2205                 return;
2206
2207         spin_lock(&file_priv->mm.lock);
2208         list_del(&request->client_list);
2209         request->file_priv = NULL;
2210         spin_unlock(&file_priv->mm.lock);
2211 }
2212
2213 static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
2214                                     struct i915_address_space *vm)
2215 {
2216         if (acthd >= i915_gem_obj_offset(obj, vm) &&
2217             acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
2218                 return true;
2219
2220         return false;
2221 }
2222
2223 static bool i915_head_inside_request(const u32 acthd_unmasked,
2224                                      const u32 request_start,
2225                                      const u32 request_end)
2226 {
2227         const u32 acthd = acthd_unmasked & HEAD_ADDR;
2228
2229         if (request_start < request_end) {
2230                 if (acthd >= request_start && acthd < request_end)
2231                         return true;
2232         } else if (request_start > request_end) {
2233                 if (acthd >= request_start || acthd < request_end)
2234                         return true;
2235         }
2236
2237         return false;
2238 }
2239
2240 static struct i915_address_space *
2241 request_to_vm(struct drm_i915_gem_request *request)
2242 {
2243         struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
2244         struct i915_address_space *vm;
2245
2246         vm = &dev_priv->gtt.base;
2247
2248         return vm;
2249 }
2250
2251 static bool i915_request_guilty(struct drm_i915_gem_request *request,
2252                                 const u32 acthd, bool *inside)
2253 {
2254         /* There is a possibility that unmasked head address
2255          * pointing inside the ring, matches the batch_obj address range.
2256          * However this is extremely unlikely.
2257          */
2258         if (request->batch_obj) {
2259                 if (i915_head_inside_object(acthd, request->batch_obj,
2260                                             request_to_vm(request))) {
2261                         *inside = true;
2262                         return true;
2263                 }
2264         }
2265
2266         if (i915_head_inside_request(acthd, request->head, request->tail)) {
2267                 *inside = false;
2268                 return true;
2269         }
2270
2271         return false;
2272 }
2273
2274 static bool i915_context_is_banned(const struct i915_ctx_hang_stats *hs)
2275 {
2276         const unsigned long elapsed = get_seconds() - hs->guilty_ts;
2277
2278         if (hs->banned)
2279                 return true;
2280
2281         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2282                 DRM_ERROR("context hanging too fast, declaring banned!\n");
2283                 return true;
2284         }
2285
2286         return false;
2287 }
2288
2289 static void i915_set_reset_status(struct intel_ring_buffer *ring,
2290                                   struct drm_i915_gem_request *request,
2291                                   u32 acthd)
2292 {
2293         struct i915_ctx_hang_stats *hs = NULL;
2294         bool inside, guilty;
2295         unsigned long offset = 0;
2296
2297         /* Innocent until proven guilty */
2298         guilty = false;
2299
2300         if (request->batch_obj)
2301                 offset = i915_gem_obj_offset(request->batch_obj,
2302                                              request_to_vm(request));
2303
2304         if (ring->hangcheck.action != HANGCHECK_WAIT &&
2305             i915_request_guilty(request, acthd, &inside)) {
2306                 DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
2307                           ring->name,
2308                           inside ? "inside" : "flushing",
2309                           offset,
2310                           request->ctx ? request->ctx->id : 0,
2311                           acthd);
2312
2313                 guilty = true;
2314         }
2315
2316         /* If contexts are disabled or this is the default context, use
2317          * file_priv->reset_state
2318          */
2319         if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
2320                 hs = &request->ctx->hang_stats;
2321         else if (request->file_priv)
2322                 hs = &request->file_priv->hang_stats;
2323
2324         if (hs) {
2325                 if (guilty) {
2326                         hs->banned = i915_context_is_banned(hs);
2327                         hs->batch_active++;
2328                         hs->guilty_ts = get_seconds();
2329                 } else {
2330                         hs->batch_pending++;
2331                 }
2332         }
2333 }
2334
2335 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2336 {
2337         list_del(&request->list);
2338         i915_gem_request_remove_from_client(request);
2339
2340         if (request->ctx)
2341                 i915_gem_context_unreference(request->ctx);
2342
2343         kfree(request);
2344 }
2345
2346 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2347                                       struct intel_ring_buffer *ring)
2348 {
2349         u32 completed_seqno;
2350         u32 acthd;
2351
2352         acthd = intel_ring_get_active_head(ring);
2353         completed_seqno = ring->get_seqno(ring, false);
2354
2355         while (!list_empty(&ring->request_list)) {
2356                 struct drm_i915_gem_request *request;
2357
2358                 request = list_first_entry(&ring->request_list,
2359                                            struct drm_i915_gem_request,
2360                                            list);
2361
2362                 if (request->seqno > completed_seqno)
2363                         i915_set_reset_status(ring, request, acthd);
2364
2365                 i915_gem_free_request(request);
2366         }
2367
2368         while (!list_empty(&ring->active_list)) {
2369                 struct drm_i915_gem_object *obj;
2370
2371                 obj = list_first_entry(&ring->active_list,
2372                                        struct drm_i915_gem_object,
2373                                        ring_list);
2374
2375                 i915_gem_object_move_to_inactive(obj);
2376         }
2377 }
2378
2379 void i915_gem_restore_fences(struct drm_device *dev)
2380 {
2381         struct drm_i915_private *dev_priv = dev->dev_private;
2382         int i;
2383
2384         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2385                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2386
2387                 /*
2388                  * Commit delayed tiling changes if we have an object still
2389                  * attached to the fence, otherwise just clear the fence.
2390                  */
2391                 if (reg->obj) {
2392                         i915_gem_object_update_fence(reg->obj, reg,
2393                                                      reg->obj->tiling_mode);
2394                 } else {
2395                         i915_gem_write_fence(dev, i, NULL);
2396                 }
2397         }
2398 }
2399
2400 void i915_gem_reset(struct drm_device *dev)
2401 {
2402         struct drm_i915_private *dev_priv = dev->dev_private;
2403         struct intel_ring_buffer *ring;
2404         int i;
2405
2406         for_each_ring(ring, dev_priv, i)
2407                 i915_gem_reset_ring_lists(dev_priv, ring);
2408
2409         i915_gem_cleanup_ringbuffer(dev);
2410
2411         i915_gem_restore_fences(dev);
2412 }
2413
2414 /**
2415  * This function clears the request list as sequence numbers are passed.
2416  */
2417 void
2418 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2419 {
2420         uint32_t seqno;
2421
2422         if (list_empty(&ring->request_list))
2423                 return;
2424
2425         WARN_ON(i915_verify_lists(ring->dev));
2426
2427         seqno = ring->get_seqno(ring, true);
2428
2429         while (!list_empty(&ring->request_list)) {
2430                 struct drm_i915_gem_request *request;
2431
2432                 request = list_first_entry(&ring->request_list,
2433                                            struct drm_i915_gem_request,
2434                                            list);
2435
2436                 if (!i915_seqno_passed(seqno, request->seqno))
2437                         break;
2438
2439                 trace_i915_gem_request_retire(ring, request->seqno);
2440                 /* We know the GPU must have read the request to have
2441                  * sent us the seqno + interrupt, so use the position
2442                  * of tail of the request to update the last known position
2443                  * of the GPU head.
2444                  */
2445                 ring->last_retired_head = request->tail;
2446
2447                 i915_gem_free_request(request);
2448         }
2449
2450         /* Move any buffers on the active list that are no longer referenced
2451          * by the ringbuffer to the flushing/inactive lists as appropriate.
2452          */
2453         while (!list_empty(&ring->active_list)) {
2454                 struct drm_i915_gem_object *obj;
2455
2456                 obj = list_first_entry(&ring->active_list,
2457                                       struct drm_i915_gem_object,
2458                                       ring_list);
2459
2460                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2461                         break;
2462
2463                 i915_gem_object_move_to_inactive(obj);
2464         }
2465
2466         if (unlikely(ring->trace_irq_seqno &&
2467                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2468                 ring->irq_put(ring);
2469                 ring->trace_irq_seqno = 0;
2470         }
2471
2472         WARN_ON(i915_verify_lists(ring->dev));
2473 }
2474
2475 bool
2476 i915_gem_retire_requests(struct drm_device *dev)
2477 {
2478         drm_i915_private_t *dev_priv = dev->dev_private;
2479         struct intel_ring_buffer *ring;
2480         bool idle = true;
2481         int i;
2482
2483         for_each_ring(ring, dev_priv, i) {
2484                 i915_gem_retire_requests_ring(ring);
2485                 idle &= list_empty(&ring->request_list);
2486         }
2487
2488         if (idle)
2489                 mod_delayed_work(dev_priv->wq,
2490                                    &dev_priv->mm.idle_work,
2491                                    msecs_to_jiffies(100));
2492
2493         return idle;
2494 }
2495
2496 static void
2497 i915_gem_retire_work_handler(struct work_struct *work)
2498 {
2499         struct drm_i915_private *dev_priv =
2500                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2501         struct drm_device *dev = dev_priv->dev;
2502         bool idle;
2503
2504         /* Come back later if the device is busy... */
2505         idle = false;
2506         if (mutex_trylock(&dev->struct_mutex)) {
2507                 idle = i915_gem_retire_requests(dev);
2508                 mutex_unlock(&dev->struct_mutex);
2509         }
2510         if (!idle)
2511                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2512                                    round_jiffies_up_relative(HZ));
2513 }
2514
2515 static void
2516 i915_gem_idle_work_handler(struct work_struct *work)
2517 {
2518         struct drm_i915_private *dev_priv =
2519                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2520
2521         intel_mark_idle(dev_priv->dev);
2522 }
2523
2524 /**
2525  * Ensures that an object will eventually get non-busy by flushing any required
2526  * write domains, emitting any outstanding lazy request and retiring and
2527  * completed requests.
2528  */
2529 static int
2530 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2531 {
2532         int ret;
2533
2534         if (obj->active) {
2535                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2536                 if (ret)
2537                         return ret;
2538
2539                 i915_gem_retire_requests_ring(obj->ring);
2540         }
2541
2542         return 0;
2543 }
2544
2545 /**
2546  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2547  * @DRM_IOCTL_ARGS: standard ioctl arguments
2548  *
2549  * Returns 0 if successful, else an error is returned with the remaining time in
2550  * the timeout parameter.
2551  *  -ETIME: object is still busy after timeout
2552  *  -ERESTARTSYS: signal interrupted the wait
2553  *  -ENONENT: object doesn't exist
2554  * Also possible, but rare:
2555  *  -EAGAIN: GPU wedged
2556  *  -ENOMEM: damn
2557  *  -ENODEV: Internal IRQ fail
2558  *  -E?: The add request failed
2559  *
2560  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2561  * non-zero timeout parameter the wait ioctl will wait for the given number of
2562  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2563  * without holding struct_mutex the object may become re-busied before this
2564  * function completes. A similar but shorter * race condition exists in the busy
2565  * ioctl
2566  */
2567 int
2568 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2569 {
2570         drm_i915_private_t *dev_priv = dev->dev_private;
2571         struct drm_i915_gem_wait *args = data;
2572         struct drm_i915_gem_object *obj;
2573         struct intel_ring_buffer *ring = NULL;
2574         struct timespec timeout_stack, *timeout = NULL;
2575         unsigned reset_counter;
2576         u32 seqno = 0;
2577         int ret = 0;
2578
2579         if (args->timeout_ns >= 0) {
2580                 timeout_stack = ns_to_timespec(args->timeout_ns);
2581                 timeout = &timeout_stack;
2582         }
2583
2584         ret = i915_mutex_lock_interruptible(dev);
2585         if (ret)
2586                 return ret;
2587
2588         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2589         if (&obj->base == NULL) {
2590                 mutex_unlock(&dev->struct_mutex);
2591                 return -ENOENT;
2592         }
2593
2594         /* Need to make sure the object gets inactive eventually. */
2595         ret = i915_gem_object_flush_active(obj);
2596         if (ret)
2597                 goto out;
2598
2599         if (obj->active) {
2600                 seqno = obj->last_read_seqno;
2601                 ring = obj->ring;
2602         }
2603
2604         if (seqno == 0)
2605                  goto out;
2606
2607         /* Do this after OLR check to make sure we make forward progress polling
2608          * on this IOCTL with a 0 timeout (like busy ioctl)
2609          */
2610         if (!args->timeout_ns) {
2611                 ret = -ETIME;
2612                 goto out;
2613         }
2614
2615         drm_gem_object_unreference(&obj->base);
2616         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2617         mutex_unlock(&dev->struct_mutex);
2618
2619         ret = __wait_seqno(ring, seqno, reset_counter, true, timeout, file->driver_priv);
2620         if (timeout)
2621                 args->timeout_ns = timespec_to_ns(timeout);
2622         return ret;
2623
2624 out:
2625         drm_gem_object_unreference(&obj->base);
2626         mutex_unlock(&dev->struct_mutex);
2627         return ret;
2628 }
2629
2630 /**
2631  * i915_gem_object_sync - sync an object to a ring.
2632  *
2633  * @obj: object which may be in use on another ring.
2634  * @to: ring we wish to use the object on. May be NULL.
2635  *
2636  * This code is meant to abstract object synchronization with the GPU.
2637  * Calling with NULL implies synchronizing the object with the CPU
2638  * rather than a particular GPU ring.
2639  *
2640  * Returns 0 if successful, else propagates up the lower layer error.
2641  */
2642 int
2643 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2644                      struct intel_ring_buffer *to)
2645 {
2646         struct intel_ring_buffer *from = obj->ring;
2647         u32 seqno;
2648         int ret, idx;
2649
2650         if (from == NULL || to == from)
2651                 return 0;
2652
2653         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2654                 return i915_gem_object_wait_rendering(obj, false);
2655
2656         idx = intel_ring_sync_index(from, to);
2657
2658         seqno = obj->last_read_seqno;
2659         if (seqno <= from->sync_seqno[idx])
2660                 return 0;
2661
2662         ret = i915_gem_check_olr(obj->ring, seqno);
2663         if (ret)
2664                 return ret;
2665
2666         trace_i915_gem_ring_sync_to(from, to, seqno);
2667         ret = to->sync_to(to, from, seqno);
2668         if (!ret)
2669                 /* We use last_read_seqno because sync_to()
2670                  * might have just caused seqno wrap under
2671                  * the radar.
2672                  */
2673                 from->sync_seqno[idx] = obj->last_read_seqno;
2674
2675         return ret;
2676 }
2677
2678 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2679 {
2680         u32 old_write_domain, old_read_domains;
2681
2682         /* Force a pagefault for domain tracking on next user access */
2683         i915_gem_release_mmap(obj);
2684
2685         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2686                 return;
2687
2688         /* Wait for any direct GTT access to complete */
2689         mb();
2690
2691         old_read_domains = obj->base.read_domains;
2692         old_write_domain = obj->base.write_domain;
2693
2694         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2695         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2696
2697         trace_i915_gem_object_change_domain(obj,
2698                                             old_read_domains,
2699                                             old_write_domain);
2700 }
2701
2702 int i915_vma_unbind(struct i915_vma *vma)
2703 {
2704         struct drm_i915_gem_object *obj = vma->obj;
2705         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2706         int ret;
2707
2708         /* For now we only ever use 1 vma per object */
2709         WARN_ON(!list_is_singular(&obj->vma_list));
2710
2711         if (list_empty(&vma->vma_link))
2712                 return 0;
2713
2714         if (!drm_mm_node_allocated(&vma->node)) {
2715                 i915_gem_vma_destroy(vma);
2716
2717                 return 0;
2718         }
2719
2720         if (obj->pin_count)
2721                 return -EBUSY;
2722
2723         BUG_ON(obj->pages == NULL);
2724
2725         ret = i915_gem_object_finish_gpu(obj);
2726         if (ret)
2727                 return ret;
2728         /* Continue on if we fail due to EIO, the GPU is hung so we
2729          * should be safe and we need to cleanup or else we might
2730          * cause memory corruption through use-after-free.
2731          */
2732
2733         i915_gem_object_finish_gtt(obj);
2734
2735         /* release the fence reg _after_ flushing */
2736         ret = i915_gem_object_put_fence(obj);
2737         if (ret)
2738                 return ret;
2739
2740         trace_i915_vma_unbind(vma);
2741
2742         if (obj->has_global_gtt_mapping)
2743                 i915_gem_gtt_unbind_object(obj);
2744         if (obj->has_aliasing_ppgtt_mapping) {
2745                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2746                 obj->has_aliasing_ppgtt_mapping = 0;
2747         }
2748         i915_gem_gtt_finish_object(obj);
2749         i915_gem_object_unpin_pages(obj);
2750
2751         list_del(&vma->mm_list);
2752         /* Avoid an unnecessary call to unbind on rebind. */
2753         if (i915_is_ggtt(vma->vm))
2754                 obj->map_and_fenceable = true;
2755
2756         drm_mm_remove_node(&vma->node);
2757
2758         i915_gem_vma_destroy(vma);
2759
2760         /* Since the unbound list is global, only move to that list if
2761          * no more VMAs exist. */
2762         if (list_empty(&obj->vma_list))
2763                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2764
2765         return 0;
2766 }
2767
2768 /**
2769  * Unbinds an object from the global GTT aperture.
2770  */
2771 int
2772 i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
2773 {
2774         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2775         struct i915_address_space *ggtt = &dev_priv->gtt.base;
2776
2777         if (!i915_gem_obj_ggtt_bound(obj))
2778                 return 0;
2779
2780         if (obj->pin_count)
2781                 return -EBUSY;
2782
2783         BUG_ON(obj->pages == NULL);
2784
2785         return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
2786 }
2787
2788 int i915_gpu_idle(struct drm_device *dev)
2789 {
2790         drm_i915_private_t *dev_priv = dev->dev_private;
2791         struct intel_ring_buffer *ring;
2792         int ret, i;
2793
2794         /* Flush everything onto the inactive list. */
2795         for_each_ring(ring, dev_priv, i) {
2796                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2797                 if (ret)
2798                         return ret;
2799
2800                 ret = intel_ring_idle(ring);
2801                 if (ret)
2802                         return ret;
2803         }
2804
2805         return 0;
2806 }
2807
2808 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2809                                  struct drm_i915_gem_object *obj)
2810 {
2811         drm_i915_private_t *dev_priv = dev->dev_private;
2812         int fence_reg;
2813         int fence_pitch_shift;
2814
2815         if (INTEL_INFO(dev)->gen >= 6) {
2816                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
2817                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
2818         } else {
2819                 fence_reg = FENCE_REG_965_0;
2820                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
2821         }
2822
2823         fence_reg += reg * 8;
2824
2825         /* To w/a incoherency with non-atomic 64-bit register updates,
2826          * we split the 64-bit update into two 32-bit writes. In order
2827          * for a partial fence not to be evaluated between writes, we
2828          * precede the update with write to turn off the fence register,
2829          * and only enable the fence as the last step.
2830          *
2831          * For extra levels of paranoia, we make sure each step lands
2832          * before applying the next step.
2833          */
2834         I915_WRITE(fence_reg, 0);
2835         POSTING_READ(fence_reg);
2836
2837         if (obj) {
2838                 u32 size = i915_gem_obj_ggtt_size(obj);
2839                 uint64_t val;
2840
2841                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2842                                  0xfffff000) << 32;
2843                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2844                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2845                 if (obj->tiling_mode == I915_TILING_Y)
2846                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2847                 val |= I965_FENCE_REG_VALID;
2848
2849                 I915_WRITE(fence_reg + 4, val >> 32);
2850                 POSTING_READ(fence_reg + 4);
2851
2852                 I915_WRITE(fence_reg + 0, val);
2853                 POSTING_READ(fence_reg);
2854         } else {
2855                 I915_WRITE(fence_reg + 4, 0);
2856                 POSTING_READ(fence_reg + 4);
2857         }
2858 }
2859
2860 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2861                                  struct drm_i915_gem_object *obj)
2862 {
2863         drm_i915_private_t *dev_priv = dev->dev_private;
2864         u32 val;
2865
2866         if (obj) {
2867                 u32 size = i915_gem_obj_ggtt_size(obj);
2868                 int pitch_val;
2869                 int tile_width;
2870
2871                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2872                      (size & -size) != size ||
2873                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2874                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2875                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2876
2877                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2878                         tile_width = 128;
2879                 else
2880                         tile_width = 512;
2881
2882                 /* Note: pitch better be a power of two tile widths */
2883                 pitch_val = obj->stride / tile_width;
2884                 pitch_val = ffs(pitch_val) - 1;
2885
2886                 val = i915_gem_obj_ggtt_offset(obj);
2887                 if (obj->tiling_mode == I915_TILING_Y)
2888                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2889                 val |= I915_FENCE_SIZE_BITS(size);
2890                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2891                 val |= I830_FENCE_REG_VALID;
2892         } else
2893                 val = 0;
2894
2895         if (reg < 8)
2896                 reg = FENCE_REG_830_0 + reg * 4;
2897         else
2898                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2899
2900         I915_WRITE(reg, val);
2901         POSTING_READ(reg);
2902 }
2903
2904 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2905                                 struct drm_i915_gem_object *obj)
2906 {
2907         drm_i915_private_t *dev_priv = dev->dev_private;
2908         uint32_t val;
2909
2910         if (obj) {
2911                 u32 size = i915_gem_obj_ggtt_size(obj);
2912                 uint32_t pitch_val;
2913
2914                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2915                      (size & -size) != size ||
2916                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2917                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
2918                      i915_gem_obj_ggtt_offset(obj), size);
2919
2920                 pitch_val = obj->stride / 128;
2921                 pitch_val = ffs(pitch_val) - 1;
2922
2923                 val = i915_gem_obj_ggtt_offset(obj);
2924                 if (obj->tiling_mode == I915_TILING_Y)
2925                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2926                 val |= I830_FENCE_SIZE_BITS(size);
2927                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2928                 val |= I830_FENCE_REG_VALID;
2929         } else
2930                 val = 0;
2931
2932         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2933         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2934 }
2935
2936 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
2937 {
2938         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
2939 }
2940
2941 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2942                                  struct drm_i915_gem_object *obj)
2943 {
2944         struct drm_i915_private *dev_priv = dev->dev_private;
2945
2946         /* Ensure that all CPU reads are completed before installing a fence
2947          * and all writes before removing the fence.
2948          */
2949         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
2950                 mb();
2951
2952         WARN(obj && (!obj->stride || !obj->tiling_mode),
2953              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
2954              obj->stride, obj->tiling_mode);
2955
2956         switch (INTEL_INFO(dev)->gen) {
2957         case 8:
2958         case 7:
2959         case 6:
2960         case 5:
2961         case 4: i965_write_fence_reg(dev, reg, obj); break;
2962         case 3: i915_write_fence_reg(dev, reg, obj); break;
2963         case 2: i830_write_fence_reg(dev, reg, obj); break;
2964         default: BUG();
2965         }
2966
2967         /* And similarly be paranoid that no direct access to this region
2968          * is reordered to before the fence is installed.
2969          */
2970         if (i915_gem_object_needs_mb(obj))
2971                 mb();
2972 }
2973
2974 static inline int fence_number(struct drm_i915_private *dev_priv,
2975                                struct drm_i915_fence_reg *fence)
2976 {
2977         return fence - dev_priv->fence_regs;
2978 }
2979
2980 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2981                                          struct drm_i915_fence_reg *fence,
2982                                          bool enable)
2983 {
2984         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2985         int reg = fence_number(dev_priv, fence);
2986
2987         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2988
2989         if (enable) {
2990                 obj->fence_reg = reg;
2991                 fence->obj = obj;
2992                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2993         } else {
2994                 obj->fence_reg = I915_FENCE_REG_NONE;
2995                 fence->obj = NULL;
2996                 list_del_init(&fence->lru_list);
2997         }
2998         obj->fence_dirty = false;
2999 }
3000
3001 static int
3002 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3003 {
3004         if (obj->last_fenced_seqno) {
3005                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
3006                 if (ret)
3007                         return ret;
3008
3009                 obj->last_fenced_seqno = 0;
3010         }
3011
3012         obj->fenced_gpu_access = false;
3013         return 0;
3014 }
3015
3016 int
3017 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3018 {
3019         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3020         struct drm_i915_fence_reg *fence;
3021         int ret;
3022
3023         ret = i915_gem_object_wait_fence(obj);
3024         if (ret)
3025                 return ret;
3026
3027         if (obj->fence_reg == I915_FENCE_REG_NONE)
3028                 return 0;
3029
3030         fence = &dev_priv->fence_regs[obj->fence_reg];
3031
3032         i915_gem_object_fence_lost(obj);
3033         i915_gem_object_update_fence(obj, fence, false);
3034
3035         return 0;
3036 }
3037
3038 static struct drm_i915_fence_reg *
3039 i915_find_fence_reg(struct drm_device *dev)
3040 {
3041         struct drm_i915_private *dev_priv = dev->dev_private;
3042         struct drm_i915_fence_reg *reg, *avail;
3043         int i;
3044
3045         /* First try to find a free reg */
3046         avail = NULL;
3047         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3048                 reg = &dev_priv->fence_regs[i];
3049                 if (!reg->obj)
3050                         return reg;
3051
3052                 if (!reg->pin_count)
3053                         avail = reg;
3054         }
3055
3056         if (avail == NULL)
3057                 return NULL;
3058
3059         /* None available, try to steal one or wait for a user to finish */
3060         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3061                 if (reg->pin_count)
3062                         continue;
3063
3064                 return reg;
3065         }
3066
3067         return NULL;
3068 }
3069
3070 /**
3071  * i915_gem_object_get_fence - set up fencing for an object
3072  * @obj: object to map through a fence reg
3073  *
3074  * When mapping objects through the GTT, userspace wants to be able to write
3075  * to them without having to worry about swizzling if the object is tiled.
3076  * This function walks the fence regs looking for a free one for @obj,
3077  * stealing one if it can't find any.
3078  *
3079  * It then sets up the reg based on the object's properties: address, pitch
3080  * and tiling format.
3081  *
3082  * For an untiled surface, this removes any existing fence.
3083  */
3084 int
3085 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3086 {
3087         struct drm_device *dev = obj->base.dev;
3088         struct drm_i915_private *dev_priv = dev->dev_private;
3089         bool enable = obj->tiling_mode != I915_TILING_NONE;
3090         struct drm_i915_fence_reg *reg;
3091         int ret;
3092
3093         /* Have we updated the tiling parameters upon the object and so
3094          * will need to serialise the write to the associated fence register?
3095          */
3096         if (obj->fence_dirty) {
3097                 ret = i915_gem_object_wait_fence(obj);
3098                 if (ret)
3099                         return ret;
3100         }
3101
3102         /* Just update our place in the LRU if our fence is getting reused. */
3103         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3104                 reg = &dev_priv->fence_regs[obj->fence_reg];
3105                 if (!obj->fence_dirty) {
3106                         list_move_tail(&reg->lru_list,
3107                                        &dev_priv->mm.fence_list);
3108                         return 0;
3109                 }
3110         } else if (enable) {
3111                 reg = i915_find_fence_reg(dev);
3112                 if (reg == NULL)
3113                         return -EDEADLK;
3114
3115                 if (reg->obj) {
3116                         struct drm_i915_gem_object *old = reg->obj;
3117
3118                         ret = i915_gem_object_wait_fence(old);
3119                         if (ret)
3120                                 return ret;
3121
3122                         i915_gem_object_fence_lost(old);
3123                 }
3124         } else
3125                 return 0;
3126
3127         i915_gem_object_update_fence(obj, reg, enable);
3128
3129         return 0;
3130 }
3131
3132 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
3133                                      struct drm_mm_node *gtt_space,
3134                                      unsigned long cache_level)
3135 {
3136         struct drm_mm_node *other;
3137
3138         /* On non-LLC machines we have to be careful when putting differing
3139          * types of snoopable memory together to avoid the prefetcher
3140          * crossing memory domains and dying.
3141          */
3142         if (HAS_LLC(dev))
3143                 return true;
3144
3145         if (!drm_mm_node_allocated(gtt_space))
3146                 return true;
3147
3148         if (list_empty(&gtt_space->node_list))
3149                 return true;
3150
3151         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3152         if (other->allocated && !other->hole_follows && other->color != cache_level)
3153                 return false;
3154
3155         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3156         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3157                 return false;
3158
3159         return true;
3160 }
3161
3162 static void i915_gem_verify_gtt(struct drm_device *dev)
3163 {
3164 #if WATCH_GTT
3165         struct drm_i915_private *dev_priv = dev->dev_private;
3166         struct drm_i915_gem_object *obj;
3167         int err = 0;
3168
3169         list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3170                 if (obj->gtt_space == NULL) {
3171                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
3172                         err++;
3173                         continue;
3174                 }
3175
3176                 if (obj->cache_level != obj->gtt_space->color) {
3177                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3178                                i915_gem_obj_ggtt_offset(obj),
3179                                i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3180                                obj->cache_level,
3181                                obj->gtt_space->color);
3182                         err++;
3183                         continue;
3184                 }
3185
3186                 if (!i915_gem_valid_gtt_space(dev,
3187                                               obj->gtt_space,
3188                                               obj->cache_level)) {
3189                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3190                                i915_gem_obj_ggtt_offset(obj),
3191                                i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3192                                obj->cache_level);
3193                         err++;
3194                         continue;
3195                 }
3196         }
3197
3198         WARN_ON(err);
3199 #endif
3200 }
3201
3202 /**
3203  * Finds free space in the GTT aperture and binds the object there.
3204  */
3205 static int
3206 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3207                            struct i915_address_space *vm,
3208                            unsigned alignment,
3209                            bool map_and_fenceable,
3210                            bool nonblocking)
3211 {
3212         struct drm_device *dev = obj->base.dev;
3213         drm_i915_private_t *dev_priv = dev->dev_private;
3214         u32 size, fence_size, fence_alignment, unfenced_alignment;
3215         size_t gtt_max =
3216                 map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
3217         struct i915_vma *vma;
3218         int ret;
3219
3220         fence_size = i915_gem_get_gtt_size(dev,
3221                                            obj->base.size,
3222                                            obj->tiling_mode);
3223         fence_alignment = i915_gem_get_gtt_alignment(dev,
3224                                                      obj->base.size,
3225                                                      obj->tiling_mode, true);
3226         unfenced_alignment =
3227                 i915_gem_get_gtt_alignment(dev,
3228                                                     obj->base.size,
3229                                                     obj->tiling_mode, false);
3230
3231         if (alignment == 0)
3232                 alignment = map_and_fenceable ? fence_alignment :
3233                                                 unfenced_alignment;
3234         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3235                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
3236                 return -EINVAL;
3237         }
3238
3239         size = map_and_fenceable ? fence_size : obj->base.size;
3240
3241         /* If the object is bigger than the entire aperture, reject it early
3242          * before evicting everything in a vain attempt to find space.
3243          */
3244         if (obj->base.size > gtt_max) {
3245                 DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3246                           obj->base.size,
3247                           map_and_fenceable ? "mappable" : "total",
3248                           gtt_max);
3249                 return -E2BIG;
3250         }
3251
3252         ret = i915_gem_object_get_pages(obj);
3253         if (ret)
3254                 return ret;
3255
3256         i915_gem_object_pin_pages(obj);
3257
3258         BUG_ON(!i915_is_ggtt(vm));
3259
3260         vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
3261         if (IS_ERR(vma)) {
3262                 ret = PTR_ERR(vma);
3263                 goto err_unpin;
3264         }
3265
3266         /* For now we only ever use 1 vma per object */
3267         WARN_ON(!list_is_singular(&obj->vma_list));
3268
3269 search_free:
3270         ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3271                                                   size, alignment,
3272                                                   obj->cache_level, 0, gtt_max,
3273                                                   DRM_MM_SEARCH_DEFAULT);
3274         if (ret) {
3275                 ret = i915_gem_evict_something(dev, vm, size, alignment,
3276                                                obj->cache_level,
3277                                                map_and_fenceable,
3278                                                nonblocking);
3279                 if (ret == 0)
3280                         goto search_free;
3281
3282                 goto err_free_vma;
3283         }
3284         if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
3285                                               obj->cache_level))) {
3286                 ret = -EINVAL;
3287                 goto err_remove_node;
3288         }
3289
3290         ret = i915_gem_gtt_prepare_object(obj);
3291         if (ret)
3292                 goto err_remove_node;
3293
3294         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3295         list_add_tail(&vma->mm_list, &vm->inactive_list);
3296
3297         if (i915_is_ggtt(vm)) {
3298                 bool mappable, fenceable;
3299
3300                 fenceable = (vma->node.size == fence_size &&
3301                              (vma->node.start & (fence_alignment - 1)) == 0);
3302
3303                 mappable = (vma->node.start + obj->base.size <=
3304                             dev_priv->gtt.mappable_end);
3305
3306                 obj->map_and_fenceable = mappable && fenceable;
3307         }
3308
3309         WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
3310
3311         trace_i915_vma_bind(vma, map_and_fenceable);
3312         i915_gem_verify_gtt(dev);
3313         return 0;
3314
3315 err_remove_node:
3316         drm_mm_remove_node(&vma->node);
3317 err_free_vma:
3318         i915_gem_vma_destroy(vma);
3319 err_unpin:
3320         i915_gem_object_unpin_pages(obj);
3321         return ret;
3322 }
3323
3324 bool
3325 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3326                         bool force)
3327 {
3328         /* If we don't have a page list set up, then we're not pinned
3329          * to GPU, and we can ignore the cache flush because it'll happen
3330          * again at bind time.
3331          */
3332         if (obj->pages == NULL)
3333                 return false;
3334
3335         /*
3336          * Stolen memory is always coherent with the GPU as it is explicitly
3337          * marked as wc by the system, or the system is cache-coherent.
3338          */
3339         if (obj->stolen)
3340                 return false;
3341
3342         /* If the GPU is snooping the contents of the CPU cache,
3343          * we do not need to manually clear the CPU cache lines.  However,
3344          * the caches are only snooped when the render cache is
3345          * flushed/invalidated.  As we always have to emit invalidations
3346          * and flushes when moving into and out of the RENDER domain, correct
3347          * snooping behaviour occurs naturally as the result of our domain
3348          * tracking.
3349          */
3350         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3351                 return false;
3352
3353         trace_i915_gem_object_clflush(obj);
3354         drm_clflush_sg(obj->pages);
3355
3356         return true;
3357 }
3358
3359 /** Flushes the GTT write domain for the object if it's dirty. */
3360 static void
3361 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3362 {
3363         uint32_t old_write_domain;
3364
3365         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3366                 return;
3367
3368         /* No actual flushing is required for the GTT write domain.  Writes
3369          * to it immediately go to main memory as far as we know, so there's
3370          * no chipset flush.  It also doesn't land in render cache.
3371          *
3372          * However, we do have to enforce the order so that all writes through
3373          * the GTT land before any writes to the device, such as updates to
3374          * the GATT itself.
3375          */
3376         wmb();
3377
3378         old_write_domain = obj->base.write_domain;
3379         obj->base.write_domain = 0;
3380
3381         trace_i915_gem_object_change_domain(obj,
3382                                             obj->base.read_domains,
3383                                             old_write_domain);
3384 }
3385
3386 /** Flushes the CPU write domain for the object if it's dirty. */
3387 static void
3388 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
3389                                        bool force)
3390 {
3391         uint32_t old_write_domain;
3392
3393         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3394                 return;
3395
3396         if (i915_gem_clflush_object(obj, force))
3397                 i915_gem_chipset_flush(obj->base.dev);
3398
3399         old_write_domain = obj->base.write_domain;
3400         obj->base.write_domain = 0;
3401
3402         trace_i915_gem_object_change_domain(obj,
3403                                             obj->base.read_domains,
3404                                             old_write_domain);
3405 }
3406
3407 /**
3408  * Moves a single object to the GTT read, and possibly write domain.
3409  *
3410  * This function returns when the move is complete, including waiting on
3411  * flushes to occur.
3412  */
3413 int
3414 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3415 {
3416         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3417         uint32_t old_write_domain, old_read_domains;
3418         int ret;
3419
3420         /* Not valid to be called on unbound objects. */
3421         if (!i915_gem_obj_bound_any(obj))
3422                 return -EINVAL;
3423
3424         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3425                 return 0;
3426
3427         ret = i915_gem_object_wait_rendering(obj, !write);
3428         if (ret)
3429                 return ret;
3430
3431         i915_gem_object_flush_cpu_write_domain(obj, false);
3432
3433         /* Serialise direct access to this object with the barriers for
3434          * coherent writes from the GPU, by effectively invalidating the
3435          * GTT domain upon first access.
3436          */
3437         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3438                 mb();
3439
3440         old_write_domain = obj->base.write_domain;
3441         old_read_domains = obj->base.read_domains;
3442
3443         /* It should now be out of any other write domains, and we can update
3444          * the domain values for our changes.
3445          */
3446         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3447         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3448         if (write) {
3449                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3450                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3451                 obj->dirty = 1;
3452         }
3453
3454         trace_i915_gem_object_change_domain(obj,
3455                                             old_read_domains,
3456                                             old_write_domain);
3457
3458         /* And bump the LRU for this access */
3459         if (i915_gem_object_is_inactive(obj)) {
3460                 struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
3461                 if (vma)
3462                         list_move_tail(&vma->mm_list,
3463                                        &dev_priv->gtt.base.inactive_list);
3464
3465         }
3466
3467         return 0;
3468 }
3469
3470 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3471                                     enum i915_cache_level cache_level)
3472 {
3473         struct drm_device *dev = obj->base.dev;
3474         drm_i915_private_t *dev_priv = dev->dev_private;
3475         struct i915_vma *vma;
3476         int ret;
3477
3478         if (obj->cache_level == cache_level)
3479                 return 0;
3480
3481         if (obj->pin_count) {
3482                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3483                 return -EBUSY;
3484         }
3485
3486         list_for_each_entry(vma, &obj->vma_list, vma_link) {
3487                 if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
3488                         ret = i915_vma_unbind(vma);
3489                         if (ret)
3490                                 return ret;
3491
3492                         break;
3493                 }
3494         }
3495
3496         if (i915_gem_obj_bound_any(obj)) {
3497                 ret = i915_gem_object_finish_gpu(obj);
3498                 if (ret)
3499                         return ret;
3500
3501                 i915_gem_object_finish_gtt(obj);
3502
3503                 /* Before SandyBridge, you could not use tiling or fence
3504                  * registers with snooped memory, so relinquish any fences
3505                  * currently pointing to our region in the aperture.
3506                  */
3507                 if (INTEL_INFO(dev)->gen < 6) {
3508                         ret = i915_gem_object_put_fence(obj);
3509                         if (ret)
3510                                 return ret;
3511                 }
3512
3513                 if (obj->has_global_gtt_mapping)
3514                         i915_gem_gtt_bind_object(obj, cache_level);
3515                 if (obj->has_aliasing_ppgtt_mapping)
3516                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3517                                                obj, cache_level);
3518         }
3519
3520         list_for_each_entry(vma, &obj->vma_list, vma_link)
3521                 vma->node.color = cache_level;
3522         obj->cache_level = cache_level;
3523
3524         if (cpu_write_needs_clflush(obj)) {
3525                 u32 old_read_domains, old_write_domain;
3526
3527                 /* If we're coming from LLC cached, then we haven't
3528                  * actually been tracking whether the data is in the
3529                  * CPU cache or not, since we only allow one bit set
3530                  * in obj->write_domain and have been skipping the clflushes.
3531                  * Just set it to the CPU cache for now.
3532                  */
3533                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3534
3535                 old_read_domains = obj->base.read_domains;
3536                 old_write_domain = obj->base.write_domain;
3537
3538                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3539                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3540
3541                 trace_i915_gem_object_change_domain(obj,
3542                                                     old_read_domains,
3543                                                     old_write_domain);
3544         }
3545
3546         i915_gem_verify_gtt(dev);
3547         return 0;
3548 }
3549
3550 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3551                                struct drm_file *file)
3552 {
3553         struct drm_i915_gem_caching *args = data;
3554         struct drm_i915_gem_object *obj;
3555         int ret;
3556
3557         ret = i915_mutex_lock_interruptible(dev);
3558         if (ret)
3559                 return ret;
3560
3561         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3562         if (&obj->base == NULL) {
3563                 ret = -ENOENT;
3564                 goto unlock;
3565         }
3566
3567         switch (obj->cache_level) {
3568         case I915_CACHE_LLC:
3569         case I915_CACHE_L3_LLC:
3570                 args->caching = I915_CACHING_CACHED;
3571                 break;
3572
3573         case I915_CACHE_WT:
3574                 args->caching = I915_CACHING_DISPLAY;
3575                 break;
3576
3577         default:
3578                 args->caching = I915_CACHING_NONE;
3579                 break;
3580         }
3581
3582         drm_gem_object_unreference(&obj->base);
3583 unlock:
3584         mutex_unlock(&dev->struct_mutex);
3585         return ret;
3586 }
3587
3588 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3589                                struct drm_file *file)
3590 {
3591         struct drm_i915_gem_caching *args = data;
3592         struct drm_i915_gem_object *obj;
3593         enum i915_cache_level level;
3594         int ret;
3595
3596         switch (args->caching) {
3597         case I915_CACHING_NONE:
3598                 level = I915_CACHE_NONE;
3599                 break;
3600         case I915_CACHING_CACHED:
3601                 level = I915_CACHE_LLC;
3602                 break;
3603         case I915_CACHING_DISPLAY:
3604                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3605                 break;
3606         default:
3607                 return -EINVAL;
3608         }
3609
3610         ret = i915_mutex_lock_interruptible(dev);
3611         if (ret)
3612                 return ret;
3613
3614         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3615         if (&obj->base == NULL) {
3616                 ret = -ENOENT;
3617                 goto unlock;
3618         }
3619
3620         ret = i915_gem_object_set_cache_level(obj, level);
3621
3622         drm_gem_object_unreference(&obj->base);
3623 unlock:
3624         mutex_unlock(&dev->struct_mutex);
3625         return ret;
3626 }
3627
3628 static bool is_pin_display(struct drm_i915_gem_object *obj)
3629 {
3630         /* There are 3 sources that pin objects:
3631          *   1. The display engine (scanouts, sprites, cursors);
3632          *   2. Reservations for execbuffer;
3633          *   3. The user.
3634          *
3635          * We can ignore reservations as we hold the struct_mutex and
3636          * are only called outside of the reservation path.  The user
3637          * can only increment pin_count once, and so if after
3638          * subtracting the potential reference by the user, any pin_count
3639          * remains, it must be due to another use by the display engine.
3640          */
3641         return obj->pin_count - !!obj->user_pin_count;
3642 }
3643
3644 /*
3645  * Prepare buffer for display plane (scanout, cursors, etc).
3646  * Can be called from an uninterruptible phase (modesetting) and allows
3647  * any flushes to be pipelined (for pageflips).
3648  */
3649 int
3650 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3651                                      u32 alignment,
3652                                      struct intel_ring_buffer *pipelined)
3653 {
3654         u32 old_read_domains, old_write_domain;
3655         int ret;
3656
3657         if (pipelined != obj->ring) {
3658                 ret = i915_gem_object_sync(obj, pipelined);
3659                 if (ret)
3660                         return ret;
3661         }
3662
3663         /* Mark the pin_display early so that we account for the
3664          * display coherency whilst setting up the cache domains.
3665          */
3666         obj->pin_display = true;
3667
3668         /* The display engine is not coherent with the LLC cache on gen6.  As
3669          * a result, we make sure that the pinning that is about to occur is
3670          * done with uncached PTEs. This is lowest common denominator for all
3671          * chipsets.
3672          *
3673          * However for gen6+, we could do better by using the GFDT bit instead
3674          * of uncaching, which would allow us to flush all the LLC-cached data
3675          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3676          */
3677         ret = i915_gem_object_set_cache_level(obj,
3678                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3679         if (ret)
3680                 goto err_unpin_display;
3681
3682         /* As the user may map the buffer once pinned in the display plane
3683          * (e.g. libkms for the bootup splash), we have to ensure that we
3684          * always use map_and_fenceable for all scanout buffers.
3685          */
3686         ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
3687         if (ret)
3688                 goto err_unpin_display;
3689
3690         i915_gem_object_flush_cpu_write_domain(obj, true);
3691
3692         old_write_domain = obj->base.write_domain;
3693         old_read_domains = obj->base.read_domains;
3694
3695         /* It should now be out of any other write domains, and we can update
3696          * the domain values for our changes.
3697          */
3698         obj->base.write_domain = 0;
3699         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3700
3701         trace_i915_gem_object_change_domain(obj,
3702                                             old_read_domains,
3703                                             old_write_domain);
3704
3705         return 0;
3706
3707 err_unpin_display:
3708         obj->pin_display = is_pin_display(obj);
3709         return ret;
3710 }
3711
3712 void
3713 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
3714 {
3715         i915_gem_object_unpin(obj);
3716         obj->pin_display = is_pin_display(obj);
3717 }
3718
3719 int
3720 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3721 {
3722         int ret;
3723
3724         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3725                 return 0;
3726
3727         ret = i915_gem_object_wait_rendering(obj, false);
3728         if (ret)
3729                 return ret;
3730
3731         /* Ensure that we invalidate the GPU's caches and TLBs. */
3732         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3733         return 0;
3734 }
3735
3736 /**
3737  * Moves a single object to the CPU read, and possibly write domain.
3738  *
3739  * This function returns when the move is complete, including waiting on
3740  * flushes to occur.
3741  */
3742 int
3743 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3744 {
3745         uint32_t old_write_domain, old_read_domains;
3746         int ret;
3747
3748         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3749                 return 0;
3750
3751         ret = i915_gem_object_wait_rendering(obj, !write);
3752         if (ret)
3753                 return ret;
3754
3755         i915_gem_object_flush_gtt_write_domain(obj);
3756
3757         old_write_domain = obj->base.write_domain;
3758         old_read_domains = obj->base.read_domains;
3759
3760         /* Flush the CPU cache if it's still invalid. */
3761         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3762                 i915_gem_clflush_object(obj, false);
3763
3764                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3765         }
3766
3767         /* It should now be out of any other write domains, and we can update
3768          * the domain values for our changes.
3769          */
3770         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3771
3772         /* If we're writing through the CPU, then the GPU read domains will
3773          * need to be invalidated at next use.
3774          */
3775         if (write) {
3776                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3777                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3778         }
3779
3780         trace_i915_gem_object_change_domain(obj,
3781                                             old_read_domains,
3782                                             old_write_domain);
3783
3784         return 0;
3785 }
3786
3787 /* Throttle our rendering by waiting until the ring has completed our requests
3788  * emitted over 20 msec ago.
3789  *
3790  * Note that if we were to use the current jiffies each time around the loop,
3791  * we wouldn't escape the function with any frames outstanding if the time to
3792  * render a frame was over 20ms.
3793  *
3794  * This should get us reasonable parallelism between CPU and GPU but also
3795  * relatively low latency when blocking on a particular request to finish.
3796  */
3797 static int
3798 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3799 {
3800         struct drm_i915_private *dev_priv = dev->dev_private;
3801         struct drm_i915_file_private *file_priv = file->driver_priv;
3802         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3803         struct drm_i915_gem_request *request;
3804         struct intel_ring_buffer *ring = NULL;
3805         unsigned reset_counter;
3806         u32 seqno = 0;
3807         int ret;
3808
3809         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3810         if (ret)
3811                 return ret;
3812
3813         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
3814         if (ret)
3815                 return ret;
3816
3817         spin_lock(&file_priv->mm.lock);
3818         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3819                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3820                         break;
3821
3822                 ring = request->ring;
3823                 seqno = request->seqno;
3824         }
3825         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3826         spin_unlock(&file_priv->mm.lock);
3827
3828         if (seqno == 0)
3829                 return 0;
3830
3831         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
3832         if (ret == 0)
3833                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3834
3835         return ret;
3836 }
3837
3838 int
3839 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3840                     struct i915_address_space *vm,
3841                     uint32_t alignment,
3842                     bool map_and_fenceable,
3843                     bool nonblocking)
3844 {
3845         struct i915_vma *vma;
3846         int ret;
3847
3848         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3849                 return -EBUSY;
3850
3851         WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
3852
3853         vma = i915_gem_obj_to_vma(obj, vm);
3854
3855         if (vma) {
3856                 if ((alignment &&
3857                      vma->node.start & (alignment - 1)) ||
3858                     (map_and_fenceable && !obj->map_and_fenceable)) {
3859                         WARN(obj->pin_count,
3860                              "bo is already pinned with incorrect alignment:"
3861                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3862                              " obj->map_and_fenceable=%d\n",
3863                              i915_gem_obj_offset(obj, vm), alignment,
3864                              map_and_fenceable,
3865                              obj->map_and_fenceable);
3866                         ret = i915_vma_unbind(vma);
3867                         if (ret)
3868                                 return ret;
3869                 }
3870         }
3871
3872         if (!i915_gem_obj_bound(obj, vm)) {
3873                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3874
3875                 ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3876                                                  map_and_fenceable,
3877                                                  nonblocking);
3878                 if (ret)
3879                         return ret;
3880
3881                 if (!dev_priv->mm.aliasing_ppgtt)
3882                         i915_gem_gtt_bind_object(obj, obj->cache_level);
3883         }
3884
3885         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3886                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3887
3888         obj->pin_count++;
3889         obj->pin_mappable |= map_and_fenceable;
3890
3891         return 0;
3892 }
3893
3894 void
3895 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3896 {
3897         BUG_ON(obj->pin_count == 0);
3898         BUG_ON(!i915_gem_obj_bound_any(obj));
3899
3900         if (--obj->pin_count == 0)
3901                 obj->pin_mappable = false;
3902 }
3903
3904 int
3905 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3906                    struct drm_file *file)
3907 {
3908         struct drm_i915_gem_pin *args = data;
3909         struct drm_i915_gem_object *obj;
3910         int ret;
3911
3912         ret = i915_mutex_lock_interruptible(dev);
3913         if (ret)
3914                 return ret;
3915
3916         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3917         if (&obj->base == NULL) {
3918                 ret = -ENOENT;
3919                 goto unlock;
3920         }
3921
3922         if (obj->madv != I915_MADV_WILLNEED) {
3923                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3924                 ret = -EINVAL;
3925                 goto out;
3926         }
3927
3928         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3929                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3930                           args->handle);
3931                 ret = -EINVAL;
3932                 goto out;
3933         }
3934
3935         if (obj->user_pin_count == ULONG_MAX) {
3936                 ret = -EBUSY;
3937                 goto out;
3938         }
3939
3940         if (obj->user_pin_count == 0) {
3941                 ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3942                 if (ret)
3943                         goto out;
3944         }
3945
3946         obj->user_pin_count++;
3947         obj->pin_filp = file;
3948
3949         args->offset = i915_gem_obj_ggtt_offset(obj);
3950 out:
3951         drm_gem_object_unreference(&obj->base);
3952 unlock:
3953         mutex_unlock(&dev->struct_mutex);
3954         return ret;
3955 }
3956
3957 int
3958 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3959                      struct drm_file *file)
3960 {
3961         struct drm_i915_gem_pin *args = data;
3962         struct drm_i915_gem_object *obj;
3963         int ret;
3964
3965         ret = i915_mutex_lock_interruptible(dev);
3966         if (ret)
3967                 return ret;
3968
3969         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3970         if (&obj->base == NULL) {
3971                 ret = -ENOENT;
3972                 goto unlock;
3973         }
3974
3975         if (obj->pin_filp != file) {
3976                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3977                           args->handle);
3978                 ret = -EINVAL;
3979                 goto out;
3980         }
3981         obj->user_pin_count--;
3982         if (obj->user_pin_count == 0) {
3983                 obj->pin_filp = NULL;
3984                 i915_gem_object_unpin(obj);
3985         }
3986
3987 out:
3988         drm_gem_object_unreference(&obj->base);
3989 unlock:
3990         mutex_unlock(&dev->struct_mutex);
3991         return ret;
3992 }
3993
3994 int
3995 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3996                     struct drm_file *file)
3997 {
3998         struct drm_i915_gem_busy *args = data;
3999         struct drm_i915_gem_object *obj;
4000         int ret;
4001
4002         ret = i915_mutex_lock_interruptible(dev);
4003         if (ret)
4004                 return ret;
4005
4006         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4007         if (&obj->base == NULL) {
4008                 ret = -ENOENT;
4009                 goto unlock;
4010         }
4011
4012         /* Count all active objects as busy, even if they are currently not used
4013          * by the gpu. Users of this interface expect objects to eventually
4014          * become non-busy without any further actions, therefore emit any
4015          * necessary flushes here.
4016          */
4017         ret = i915_gem_object_flush_active(obj);
4018
4019         args->busy = obj->active;
4020         if (obj->ring) {
4021                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
4022                 args->busy |= intel_ring_flag(obj->ring) << 16;
4023         }
4024
4025         drm_gem_object_unreference(&obj->base);
4026 unlock:
4027         mutex_unlock(&dev->struct_mutex);
4028         return ret;
4029 }
4030
4031 int
4032 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4033                         struct drm_file *file_priv)
4034 {
4035         return i915_gem_ring_throttle(dev, file_priv);
4036 }
4037
4038 int
4039 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4040                        struct drm_file *file_priv)
4041 {
4042         struct drm_i915_gem_madvise *args = data;
4043         struct drm_i915_gem_object *obj;
4044         int ret;
4045
4046         switch (args->madv) {
4047         case I915_MADV_DONTNEED:
4048         case I915_MADV_WILLNEED:
4049             break;
4050         default:
4051             return -EINVAL;
4052         }
4053
4054         ret = i915_mutex_lock_interruptible(dev);
4055         if (ret)
4056                 return ret;
4057
4058         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4059         if (&obj->base == NULL) {
4060                 ret = -ENOENT;
4061                 goto unlock;
4062         }
4063
4064         if (obj->pin_count) {
4065                 ret = -EINVAL;
4066                 goto out;
4067         }
4068
4069         if (obj->madv != __I915_MADV_PURGED)
4070                 obj->madv = args->madv;
4071
4072         /* if the object is no longer attached, discard its backing storage */
4073         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
4074                 i915_gem_object_truncate(obj);
4075
4076         args->retained = obj->madv != __I915_MADV_PURGED;
4077
4078 out:
4079         drm_gem_object_unreference(&obj->base);
4080 unlock:
4081         mutex_unlock(&dev->struct_mutex);
4082         return ret;
4083 }
4084
4085 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4086                           const struct drm_i915_gem_object_ops *ops)
4087 {
4088         INIT_LIST_HEAD(&obj->global_list);
4089         INIT_LIST_HEAD(&obj->ring_list);
4090         INIT_LIST_HEAD(&obj->obj_exec_link);
4091         INIT_LIST_HEAD(&obj->vma_list);
4092
4093         obj->ops = ops;
4094
4095         obj->fence_reg = I915_FENCE_REG_NONE;
4096         obj->madv = I915_MADV_WILLNEED;
4097         /* Avoid an unnecessary call to unbind on the first bind. */
4098         obj->map_and_fenceable = true;
4099
4100         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4101 }
4102
4103 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4104         .get_pages = i915_gem_object_get_pages_gtt,
4105         .put_pages = i915_gem_object_put_pages_gtt,
4106 };
4107
4108 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4109                                                   size_t size)
4110 {
4111         struct drm_i915_gem_object *obj;
4112         struct address_space *mapping;
4113         gfp_t mask;
4114
4115         obj = i915_gem_object_alloc(dev);
4116         if (obj == NULL)
4117                 return NULL;
4118
4119         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4120                 i915_gem_object_free(obj);
4121                 return NULL;
4122         }
4123
4124         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4125         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4126                 /* 965gm cannot relocate objects above 4GiB. */
4127                 mask &= ~__GFP_HIGHMEM;
4128                 mask |= __GFP_DMA32;
4129         }
4130
4131         mapping = file_inode(obj->base.filp)->i_mapping;
4132         mapping_set_gfp_mask(mapping, mask);
4133
4134         i915_gem_object_init(obj, &i915_gem_object_ops);
4135
4136         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4137         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4138
4139         if (HAS_LLC(dev)) {
4140                 /* On some devices, we can have the GPU use the LLC (the CPU
4141                  * cache) for about a 10% performance improvement
4142                  * compared to uncached.  Graphics requests other than
4143                  * display scanout are coherent with the CPU in
4144                  * accessing this cache.  This means in this mode we
4145                  * don't need to clflush on the CPU side, and on the
4146                  * GPU side we only need to flush internal caches to
4147                  * get data visible to the CPU.
4148                  *
4149                  * However, we maintain the display planes as UC, and so
4150                  * need to rebind when first used as such.
4151                  */
4152                 obj->cache_level = I915_CACHE_LLC;
4153         } else
4154                 obj->cache_level = I915_CACHE_NONE;
4155
4156         trace_i915_gem_object_create(obj);
4157
4158         return obj;
4159 }
4160
4161 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4162 {
4163         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4164         struct drm_device *dev = obj->base.dev;
4165         drm_i915_private_t *dev_priv = dev->dev_private;
4166         struct i915_vma *vma, *next;
4167
4168         trace_i915_gem_object_destroy(obj);
4169
4170         if (obj->phys_obj)
4171                 i915_gem_detach_phys_object(dev, obj);
4172
4173         obj->pin_count = 0;
4174         /* NB: 0 or 1 elements */
4175         WARN_ON(!list_empty(&obj->vma_list) &&
4176                 !list_is_singular(&obj->vma_list));
4177         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4178                 int ret = i915_vma_unbind(vma);
4179                 if (WARN_ON(ret == -ERESTARTSYS)) {
4180                         bool was_interruptible;
4181
4182                         was_interruptible = dev_priv->mm.interruptible;
4183                         dev_priv->mm.interruptible = false;
4184
4185                         WARN_ON(i915_vma_unbind(vma));
4186
4187                         dev_priv->mm.interruptible = was_interruptible;
4188                 }
4189         }
4190
4191         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4192          * before progressing. */
4193         if (obj->stolen)
4194                 i915_gem_object_unpin_pages(obj);
4195
4196         if (WARN_ON(obj->pages_pin_count))
4197                 obj->pages_pin_count = 0;
4198         i915_gem_object_put_pages(obj);
4199         i915_gem_object_free_mmap_offset(obj);
4200         i915_gem_object_release_stolen(obj);
4201
4202         BUG_ON(obj->pages);
4203
4204         if (obj->base.import_attach)
4205                 drm_prime_gem_destroy(&obj->base, NULL);
4206
4207         drm_gem_object_release(&obj->base);
4208         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4209
4210         kfree(obj->bit_17);
4211         i915_gem_object_free(obj);
4212 }
4213
4214 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4215                                      struct i915_address_space *vm)
4216 {
4217         struct i915_vma *vma;
4218         list_for_each_entry(vma, &obj->vma_list, vma_link)
4219                 if (vma->vm == vm)
4220                         return vma;
4221
4222         return NULL;
4223 }
4224
4225 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
4226                                               struct i915_address_space *vm)
4227 {
4228         struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
4229         if (vma == NULL)
4230                 return ERR_PTR(-ENOMEM);
4231
4232         INIT_LIST_HEAD(&vma->vma_link);
4233         INIT_LIST_HEAD(&vma->mm_list);
4234         INIT_LIST_HEAD(&vma->exec_list);
4235         vma->vm = vm;
4236         vma->obj = obj;
4237
4238         /* Keep GGTT vmas first to make debug easier */
4239         if (i915_is_ggtt(vm))
4240                 list_add(&vma->vma_link, &obj->vma_list);
4241         else
4242                 list_add_tail(&vma->vma_link, &obj->vma_list);
4243
4244         return vma;
4245 }
4246
4247 struct i915_vma *
4248 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
4249                                   struct i915_address_space *vm)
4250 {
4251         struct i915_vma *vma;
4252
4253         vma = i915_gem_obj_to_vma(obj, vm);
4254         if (!vma)
4255                 vma = __i915_gem_vma_create(obj, vm);
4256
4257         return vma;
4258 }
4259
4260 void i915_gem_vma_destroy(struct i915_vma *vma)
4261 {
4262         WARN_ON(vma->node.allocated);
4263
4264         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4265         if (!list_empty(&vma->exec_list))
4266                 return;
4267
4268         list_del(&vma->vma_link);
4269
4270         kfree(vma);
4271 }
4272
4273 int
4274 i915_gem_suspend(struct drm_device *dev)
4275 {
4276         drm_i915_private_t *dev_priv = dev->dev_private;
4277         int ret = 0;
4278
4279         mutex_lock(&dev->struct_mutex);
4280         if (dev_priv->ums.mm_suspended)
4281                 goto err;
4282
4283         ret = i915_gpu_idle(dev);
4284         if (ret)
4285                 goto err;
4286
4287         i915_gem_retire_requests(dev);
4288
4289         /* Under UMS, be paranoid and evict. */
4290         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4291                 i915_gem_evict_everything(dev);
4292
4293         i915_kernel_lost_context(dev);
4294         i915_gem_cleanup_ringbuffer(dev);
4295
4296         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4297          * We need to replace this with a semaphore, or something.
4298          * And not confound ums.mm_suspended!
4299          */
4300         dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
4301                                                              DRIVER_MODESET);
4302         mutex_unlock(&dev->struct_mutex);
4303
4304         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4305         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4306         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
4307
4308         return 0;
4309
4310 err:
4311         mutex_unlock(&dev->struct_mutex);
4312         return ret;
4313 }
4314
4315 int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice)
4316 {
4317         struct drm_device *dev = ring->dev;
4318         drm_i915_private_t *dev_priv = dev->dev_private;
4319         u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
4320         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4321         int i, ret;
4322
4323         if (!HAS_L3_DPF(dev) || !remap_info)
4324                 return 0;
4325
4326         ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
4327         if (ret)
4328                 return ret;
4329
4330         /*
4331          * Note: We do not worry about the concurrent register cacheline hang
4332          * here because no other code should access these registers other than
4333          * at initialization time.
4334          */
4335         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4336                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4337                 intel_ring_emit(ring, reg_base + i);
4338                 intel_ring_emit(ring, remap_info[i/4]);
4339         }
4340
4341         intel_ring_advance(ring);
4342
4343         return ret;
4344 }
4345
4346 void i915_gem_init_swizzling(struct drm_device *dev)
4347 {
4348         drm_i915_private_t *dev_priv = dev->dev_private;
4349
4350         if (INTEL_INFO(dev)->gen < 5 ||
4351             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4352                 return;
4353
4354         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4355                                  DISP_TILE_SURFACE_SWIZZLING);
4356
4357         if (IS_GEN5(dev))
4358                 return;
4359
4360         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4361         if (IS_GEN6(dev))
4362                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4363         else if (IS_GEN7(dev))
4364                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4365         else if (IS_GEN8(dev))
4366                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4367         else
4368                 BUG();
4369 }
4370
4371 static bool
4372 intel_enable_blt(struct drm_device *dev)
4373 {
4374         if (!HAS_BLT(dev))
4375                 return false;
4376
4377         /* The blitter was dysfunctional on early prototypes */
4378         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4379                 DRM_INFO("BLT not supported on this pre-production hardware;"
4380                          " graphics performance will be degraded.\n");
4381                 return false;
4382         }
4383
4384         return true;
4385 }
4386
4387 static int i915_gem_init_rings(struct drm_device *dev)
4388 {
4389         struct drm_i915_private *dev_priv = dev->dev_private;
4390         int ret;
4391
4392         ret = intel_init_render_ring_buffer(dev);
4393         if (ret)
4394                 return ret;
4395
4396         if (HAS_BSD(dev)) {
4397                 ret = intel_init_bsd_ring_buffer(dev);
4398                 if (ret)
4399                         goto cleanup_render_ring;
4400         }
4401
4402         if (intel_enable_blt(dev)) {
4403                 ret = intel_init_blt_ring_buffer(dev);
4404                 if (ret)
4405                         goto cleanup_bsd_ring;
4406         }
4407
4408         if (HAS_VEBOX(dev)) {
4409                 ret = intel_init_vebox_ring_buffer(dev);
4410                 if (ret)
4411                         goto cleanup_blt_ring;
4412         }
4413
4414
4415         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4416         if (ret)
4417                 goto cleanup_vebox_ring;
4418
4419         return 0;
4420
4421 cleanup_vebox_ring:
4422         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4423 cleanup_blt_ring:
4424         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4425 cleanup_bsd_ring:
4426         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4427 cleanup_render_ring:
4428         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4429
4430         return ret;
4431 }
4432
4433 int
4434 i915_gem_init_hw(struct drm_device *dev)
4435 {
4436         drm_i915_private_t *dev_priv = dev->dev_private;
4437         int ret, i;
4438
4439         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4440                 return -EIO;
4441
4442         if (dev_priv->ellc_size)
4443                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4444
4445         if (IS_HASWELL(dev))
4446                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4447                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4448
4449         if (HAS_PCH_NOP(dev)) {
4450                 u32 temp = I915_READ(GEN7_MSG_CTL);
4451                 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4452                 I915_WRITE(GEN7_MSG_CTL, temp);
4453         }
4454
4455         i915_gem_init_swizzling(dev);
4456
4457         ret = i915_gem_init_rings(dev);
4458         if (ret)
4459                 return ret;
4460
4461         for (i = 0; i < NUM_L3_SLICES(dev); i++)
4462                 i915_gem_l3_remap(&dev_priv->ring[RCS], i);
4463
4464         /*
4465          * XXX: There was some w/a described somewhere suggesting loading
4466          * contexts before PPGTT.
4467          */
4468         i915_gem_context_init(dev);
4469         if (dev_priv->mm.aliasing_ppgtt) {
4470                 ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
4471                 if (ret) {
4472                         i915_gem_cleanup_aliasing_ppgtt(dev);
4473                         DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
4474                 }
4475         }
4476
4477         return 0;
4478 }
4479
4480 int i915_gem_init(struct drm_device *dev)
4481 {
4482         struct drm_i915_private *dev_priv = dev->dev_private;
4483         int ret;
4484
4485         mutex_lock(&dev->struct_mutex);
4486
4487         if (IS_VALLEYVIEW(dev)) {
4488                 /* VLVA0 (potential hack), BIOS isn't actually waking us */
4489                 I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
4490                 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
4491                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4492         }
4493
4494         i915_gem_init_global_gtt(dev);
4495
4496         ret = i915_gem_init_hw(dev);
4497         mutex_unlock(&dev->struct_mutex);
4498         if (ret) {
4499                 i915_gem_cleanup_aliasing_ppgtt(dev);
4500                 return ret;
4501         }
4502
4503         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4504         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4505                 dev_priv->dri1.allow_batchbuffer = 1;
4506         return 0;
4507 }
4508
4509 void
4510 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4511 {
4512         drm_i915_private_t *dev_priv = dev->dev_private;
4513         struct intel_ring_buffer *ring;
4514         int i;
4515
4516         for_each_ring(ring, dev_priv, i)
4517                 intel_cleanup_ring_buffer(ring);
4518 }
4519
4520 int
4521 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4522                        struct drm_file *file_priv)
4523 {
4524         struct drm_i915_private *dev_priv = dev->dev_private;
4525         int ret;
4526
4527         if (drm_core_check_feature(dev, DRIVER_MODESET))
4528                 return 0;
4529
4530         if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4531                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4532                 atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4533         }
4534
4535         mutex_lock(&dev->struct_mutex);
4536         dev_priv->ums.mm_suspended = 0;
4537
4538         ret = i915_gem_init_hw(dev);
4539         if (ret != 0) {
4540                 mutex_unlock(&dev->struct_mutex);
4541                 return ret;
4542         }
4543
4544         BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
4545         mutex_unlock(&dev->struct_mutex);
4546
4547         ret = drm_irq_install(dev);
4548         if (ret)
4549                 goto cleanup_ringbuffer;
4550
4551         return 0;
4552
4553 cleanup_ringbuffer:
4554         mutex_lock(&dev->struct_mutex);
4555         i915_gem_cleanup_ringbuffer(dev);
4556         dev_priv->ums.mm_suspended = 1;
4557         mutex_unlock(&dev->struct_mutex);
4558
4559         return ret;
4560 }
4561
4562 int
4563 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4564                        struct drm_file *file_priv)
4565 {
4566         if (drm_core_check_feature(dev, DRIVER_MODESET))
4567                 return 0;
4568
4569         drm_irq_uninstall(dev);
4570
4571         return i915_gem_suspend(dev);
4572 }
4573
4574 void
4575 i915_gem_lastclose(struct drm_device *dev)
4576 {
4577         int ret;
4578
4579         if (drm_core_check_feature(dev, DRIVER_MODESET))
4580                 return;
4581
4582         ret = i915_gem_suspend(dev);
4583         if (ret)
4584                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4585 }
4586
4587 static void
4588 init_ring_lists(struct intel_ring_buffer *ring)
4589 {
4590         INIT_LIST_HEAD(&ring->active_list);
4591         INIT_LIST_HEAD(&ring->request_list);
4592 }
4593
4594 static void i915_init_vm(struct drm_i915_private *dev_priv,
4595                          struct i915_address_space *vm)
4596 {
4597         vm->dev = dev_priv->dev;
4598         INIT_LIST_HEAD(&vm->active_list);
4599         INIT_LIST_HEAD(&vm->inactive_list);
4600         INIT_LIST_HEAD(&vm->global_link);
4601         list_add(&vm->global_link, &dev_priv->vm_list);
4602 }
4603
4604 void
4605 i915_gem_load(struct drm_device *dev)
4606 {
4607         drm_i915_private_t *dev_priv = dev->dev_private;
4608         int i;
4609
4610         dev_priv->slab =
4611                 kmem_cache_create("i915_gem_object",
4612                                   sizeof(struct drm_i915_gem_object), 0,
4613                                   SLAB_HWCACHE_ALIGN,
4614                                   NULL);
4615
4616         INIT_LIST_HEAD(&dev_priv->vm_list);
4617         i915_init_vm(dev_priv, &dev_priv->gtt.base);
4618
4619         INIT_LIST_HEAD(&dev_priv->context_list);
4620         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4621         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4622         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4623         for (i = 0; i < I915_NUM_RINGS; i++)
4624                 init_ring_lists(&dev_priv->ring[i]);
4625         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4626                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4627         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4628                           i915_gem_retire_work_handler);
4629         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
4630                           i915_gem_idle_work_handler);
4631         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4632
4633         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4634         if (IS_GEN3(dev)) {
4635                 I915_WRITE(MI_ARB_STATE,
4636                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4637         }
4638
4639         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4640
4641         /* Old X drivers will take 0-2 for front, back, depth buffers */
4642         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4643                 dev_priv->fence_reg_start = 3;
4644
4645         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
4646                 dev_priv->num_fence_regs = 32;
4647         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4648                 dev_priv->num_fence_regs = 16;
4649         else
4650                 dev_priv->num_fence_regs = 8;
4651
4652         /* Initialize fence registers to zero */
4653         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4654         i915_gem_restore_fences(dev);
4655
4656         i915_gem_detect_bit_6_swizzle(dev);
4657         init_waitqueue_head(&dev_priv->pending_flip_queue);
4658
4659         dev_priv->mm.interruptible = true;
4660
4661         dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan;
4662         dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count;
4663         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4664         register_shrinker(&dev_priv->mm.inactive_shrinker);
4665 }
4666
4667 /*
4668  * Create a physically contiguous memory object for this object
4669  * e.g. for cursor + overlay regs
4670  */
4671 static int i915_gem_init_phys_object(struct drm_device *dev,
4672                                      int id, int size, int align)
4673 {
4674         drm_i915_private_t *dev_priv = dev->dev_private;
4675         struct drm_i915_gem_phys_object *phys_obj;
4676         int ret;
4677
4678         if (dev_priv->mm.phys_objs[id - 1] || !size)
4679                 return 0;
4680
4681         phys_obj = kzalloc(sizeof(*phys_obj), GFP_KERNEL);
4682         if (!phys_obj)
4683                 return -ENOMEM;
4684
4685         phys_obj->id = id;
4686
4687         phys_obj->handle = drm_pci_alloc(dev, size, align);
4688         if (!phys_obj->handle) {
4689                 ret = -ENOMEM;
4690                 goto kfree_obj;
4691         }
4692 #ifdef CONFIG_X86
4693         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4694 #endif
4695
4696         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4697
4698         return 0;
4699 kfree_obj:
4700         kfree(phys_obj);
4701         return ret;
4702 }
4703
4704 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4705 {
4706         drm_i915_private_t *dev_priv = dev->dev_private;
4707         struct drm_i915_gem_phys_object *phys_obj;
4708
4709         if (!dev_priv->mm.phys_objs[id - 1])
4710                 return;
4711
4712         phys_obj = dev_priv->mm.phys_objs[id - 1];
4713         if (phys_obj->cur_obj) {
4714                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4715         }
4716
4717 #ifdef CONFIG_X86
4718         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4719 #endif
4720         drm_pci_free(dev, phys_obj->handle);
4721         kfree(phys_obj);
4722         dev_priv->mm.phys_objs[id - 1] = NULL;
4723 }
4724
4725 void i915_gem_free_all_phys_object(struct drm_device *dev)
4726 {
4727         int i;
4728
4729         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4730                 i915_gem_free_phys_object(dev, i);
4731 }
4732
4733 void i915_gem_detach_phys_object(struct drm_device *dev,
4734                                  struct drm_i915_gem_object *obj)
4735 {
4736         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4737         char *vaddr;
4738         int i;
4739         int page_count;
4740
4741         if (!obj->phys_obj)
4742                 return;
4743         vaddr = obj->phys_obj->handle->vaddr;
4744
4745         page_count = obj->base.size / PAGE_SIZE;
4746         for (i = 0; i < page_count; i++) {
4747                 struct page *page = shmem_read_mapping_page(mapping, i);
4748                 if (!IS_ERR(page)) {
4749                         char *dst = kmap_atomic(page);
4750                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4751                         kunmap_atomic(dst);
4752
4753                         drm_clflush_pages(&page, 1);
4754
4755                         set_page_dirty(page);
4756                         mark_page_accessed(page);
4757                         page_cache_release(page);
4758                 }
4759         }
4760         i915_gem_chipset_flush(dev);
4761
4762         obj->phys_obj->cur_obj = NULL;
4763         obj->phys_obj = NULL;
4764 }
4765
4766 int
4767 i915_gem_attach_phys_object(struct drm_device *dev,
4768                             struct drm_i915_gem_object *obj,
4769                             int id,
4770                             int align)
4771 {
4772         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4773         drm_i915_private_t *dev_priv = dev->dev_private;
4774         int ret = 0;
4775         int page_count;
4776         int i;
4777
4778         if (id > I915_MAX_PHYS_OBJECT)
4779                 return -EINVAL;
4780
4781         if (obj->phys_obj) {
4782                 if (obj->phys_obj->id == id)
4783                         return 0;
4784                 i915_gem_detach_phys_object(dev, obj);
4785         }
4786
4787         /* create a new object */
4788         if (!dev_priv->mm.phys_objs[id - 1]) {
4789                 ret = i915_gem_init_phys_object(dev, id,
4790                                                 obj->base.size, align);
4791                 if (ret) {
4792                         DRM_ERROR("failed to init phys object %d size: %zu\n",
4793                                   id, obj->base.size);
4794                         return ret;
4795                 }
4796         }
4797
4798         /* bind to the object */
4799         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4800         obj->phys_obj->cur_obj = obj;
4801
4802         page_count = obj->base.size / PAGE_SIZE;
4803
4804         for (i = 0; i < page_count; i++) {
4805                 struct page *page;
4806                 char *dst, *src;
4807
4808                 page = shmem_read_mapping_page(mapping, i);
4809                 if (IS_ERR(page))
4810                         return PTR_ERR(page);
4811
4812                 src = kmap_atomic(page);
4813                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4814                 memcpy(dst, src, PAGE_SIZE);
4815                 kunmap_atomic(src);
4816
4817                 mark_page_accessed(page);
4818                 page_cache_release(page);
4819         }
4820
4821         return 0;
4822 }
4823
4824 static int
4825 i915_gem_phys_pwrite(struct drm_device *dev,
4826                      struct drm_i915_gem_object *obj,
4827                      struct drm_i915_gem_pwrite *args,
4828                      struct drm_file *file_priv)
4829 {
4830         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4831         char __user *user_data = to_user_ptr(args->data_ptr);
4832
4833         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4834                 unsigned long unwritten;
4835
4836                 /* The physical object once assigned is fixed for the lifetime
4837                  * of the obj, so we can safely drop the lock and continue
4838                  * to access vaddr.
4839                  */
4840                 mutex_unlock(&dev->struct_mutex);
4841                 unwritten = copy_from_user(vaddr, user_data, args->size);
4842                 mutex_lock(&dev->struct_mutex);
4843                 if (unwritten)
4844                         return -EFAULT;
4845         }
4846
4847         i915_gem_chipset_flush(dev);
4848         return 0;
4849 }
4850
4851 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4852 {
4853         struct drm_i915_file_private *file_priv = file->driver_priv;
4854
4855         cancel_delayed_work_sync(&file_priv->mm.idle_work);
4856
4857         /* Clean up our request list when the client is going away, so that
4858          * later retire_requests won't dereference our soon-to-be-gone
4859          * file_priv.
4860          */
4861         spin_lock(&file_priv->mm.lock);
4862         while (!list_empty(&file_priv->mm.request_list)) {
4863                 struct drm_i915_gem_request *request;
4864
4865                 request = list_first_entry(&file_priv->mm.request_list,
4866                                            struct drm_i915_gem_request,
4867                                            client_list);
4868                 list_del(&request->client_list);
4869                 request->file_priv = NULL;
4870         }
4871         spin_unlock(&file_priv->mm.lock);
4872 }
4873
4874 static void
4875 i915_gem_file_idle_work_handler(struct work_struct *work)
4876 {
4877         struct drm_i915_file_private *file_priv =
4878                 container_of(work, typeof(*file_priv), mm.idle_work.work);
4879
4880         atomic_set(&file_priv->rps_wait_boost, false);
4881 }
4882
4883 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4884 {
4885         struct drm_i915_file_private *file_priv;
4886
4887         DRM_DEBUG_DRIVER("\n");
4888
4889         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4890         if (!file_priv)
4891                 return -ENOMEM;
4892
4893         file->driver_priv = file_priv;
4894         file_priv->dev_priv = dev->dev_private;
4895
4896         spin_lock_init(&file_priv->mm.lock);
4897         INIT_LIST_HEAD(&file_priv->mm.request_list);
4898         INIT_DELAYED_WORK(&file_priv->mm.idle_work,
4899                           i915_gem_file_idle_work_handler);
4900
4901         idr_init(&file_priv->context_idr);
4902
4903         return 0;
4904 }
4905
4906 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
4907 {
4908         if (!mutex_is_locked(mutex))
4909                 return false;
4910
4911 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
4912         return mutex->owner == task;
4913 #else
4914         /* Since UP may be pre-empted, we cannot assume that we own the lock */
4915         return false;
4916 #endif
4917 }
4918
4919 static unsigned long
4920 i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
4921 {
4922         struct drm_i915_private *dev_priv =
4923                 container_of(shrinker,
4924                              struct drm_i915_private,
4925                              mm.inactive_shrinker);
4926         struct drm_device *dev = dev_priv->dev;
4927         struct drm_i915_gem_object *obj;
4928         bool unlock = true;
4929         unsigned long count;
4930
4931         if (!mutex_trylock(&dev->struct_mutex)) {
4932                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
4933                         return 0;
4934
4935                 if (dev_priv->mm.shrinker_no_lock_stealing)
4936                         return 0;
4937
4938                 unlock = false;
4939         }
4940
4941         count = 0;
4942         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4943                 if (obj->pages_pin_count == 0)
4944                         count += obj->base.size >> PAGE_SHIFT;
4945
4946         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4947                 if (obj->active)
4948                         continue;
4949
4950                 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
4951                         count += obj->base.size >> PAGE_SHIFT;
4952         }
4953
4954         if (unlock)
4955                 mutex_unlock(&dev->struct_mutex);
4956
4957         return count;
4958 }
4959
4960 /* All the new VM stuff */
4961 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
4962                                   struct i915_address_space *vm)
4963 {
4964         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
4965         struct i915_vma *vma;
4966
4967         if (vm == &dev_priv->mm.aliasing_ppgtt->base)
4968                 vm = &dev_priv->gtt.base;
4969
4970         BUG_ON(list_empty(&o->vma_list));
4971         list_for_each_entry(vma, &o->vma_list, vma_link) {
4972                 if (vma->vm == vm)
4973                         return vma->node.start;
4974
4975         }
4976         return -1;
4977 }
4978
4979 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4980                         struct i915_address_space *vm)
4981 {
4982         struct i915_vma *vma;
4983
4984         list_for_each_entry(vma, &o->vma_list, vma_link)
4985                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4986                         return true;
4987
4988         return false;
4989 }
4990
4991 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
4992 {
4993         struct i915_vma *vma;
4994
4995         list_for_each_entry(vma, &o->vma_list, vma_link)
4996                 if (drm_mm_node_allocated(&vma->node))
4997                         return true;
4998
4999         return false;
5000 }
5001
5002 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5003                                 struct i915_address_space *vm)
5004 {
5005         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5006         struct i915_vma *vma;
5007
5008         if (vm == &dev_priv->mm.aliasing_ppgtt->base)
5009                 vm = &dev_priv->gtt.base;
5010
5011         BUG_ON(list_empty(&o->vma_list));
5012
5013         list_for_each_entry(vma, &o->vma_list, vma_link)
5014                 if (vma->vm == vm)
5015                         return vma->node.size;
5016
5017         return 0;
5018 }
5019
5020 static unsigned long
5021 i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc)
5022 {
5023         struct drm_i915_private *dev_priv =
5024                 container_of(shrinker,
5025                              struct drm_i915_private,
5026                              mm.inactive_shrinker);
5027         struct drm_device *dev = dev_priv->dev;
5028         unsigned long freed;
5029         bool unlock = true;
5030
5031         if (!mutex_trylock(&dev->struct_mutex)) {
5032                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
5033                         return SHRINK_STOP;
5034
5035                 if (dev_priv->mm.shrinker_no_lock_stealing)
5036                         return SHRINK_STOP;
5037
5038                 unlock = false;
5039         }
5040
5041         freed = i915_gem_purge(dev_priv, sc->nr_to_scan);
5042         if (freed < sc->nr_to_scan)
5043                 freed += __i915_gem_shrink(dev_priv,
5044                                            sc->nr_to_scan - freed,
5045                                            false);
5046         if (freed < sc->nr_to_scan)
5047                 freed += i915_gem_shrink_all(dev_priv);
5048
5049         if (unlock)
5050                 mutex_unlock(&dev->struct_mutex);
5051
5052         return freed;
5053 }
5054
5055 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
5056 {
5057         struct i915_vma *vma;
5058
5059         if (WARN_ON(list_empty(&obj->vma_list)))
5060                 return NULL;
5061
5062         vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5063         if (WARN_ON(vma->vm != obj_to_ggtt(obj)))
5064                 return NULL;
5065
5066         return vma;
5067 }