drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38 #include <linux/dma-buf.h>
  39
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  42                                                    bool force);
  43 static __must_check int
  44 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  45                                bool readonly);
  46 static __must_check int
  47 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
  48                            struct i915_address_space *vm,
  49                            unsigned alignment,
  50                            bool map_and_fenceable,
  51                            bool nonblocking);
  52 static int i915_gem_phys_pwrite(struct drm_device *dev,
  53                                 struct drm_i915_gem_object *obj,
  54                                 struct drm_i915_gem_pwrite *args,
  55                                 struct drm_file *file);
  56
  57 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  58                                  struct drm_i915_gem_object *obj);
  59 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  60                                          struct drm_i915_fence_reg *fence,
  61                                          bool enable);
  62
  63 static unsigned long i915_gem_inactive_count(struct shrinker *shrinker,
  64                                              struct shrink_control *sc);
  65 static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker,
  66                                             struct shrink_control *sc);
  67 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  68 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  69 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  70
  71 static bool cpu_cache_is_coherent(struct drm_device *dev,
  72                                   enum i915_cache_level level)
  73 {
  74         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  75 }
  76
  77 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  78 {
  79         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  80                 return true;
  81
  82         return obj->pin_display;
  83 }
  84
  85 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  86 {
  87         if (obj->tiling_mode)
  88                 i915_gem_release_mmap(obj);
  89
  90         /* As we do not have an associated fence register, we will force
  91          * a tiling change if we ever need to acquire one.
  92          */
  93         obj->fence_dirty = false;
  94         obj->fence_reg = I915_FENCE_REG_NONE;
  95 }
  96
  97 /* some bookkeeping */
  98 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  99                                   size_t size)
 100 {
 101         spin_lock(&dev_priv->mm.object_stat_lock);
 102         dev_priv->mm.object_count++;
 103         dev_priv->mm.object_memory += size;
 104         spin_unlock(&dev_priv->mm.object_stat_lock);
 105 }
 106
 107 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 108                                      size_t size)
 109 {
 110         spin_lock(&dev_priv->mm.object_stat_lock);
 111         dev_priv->mm.object_count--;
 112         dev_priv->mm.object_memory -= size;
 113         spin_unlock(&dev_priv->mm.object_stat_lock);
 114 }
 115
 116 static int
 117 i915_gem_wait_for_error(struct i915_gpu_error *error)
 118 {
 119         int ret;
 120
 121 #define EXIT_COND (!i915_reset_in_progress(error) || \
 122                    i915_terminally_wedged(error))
 123         if (EXIT_COND)
 124                 return 0;
 125
 126         /*
 127          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 128          * userspace. If it takes that long something really bad is going on and
 129          * we should simply try to bail out and fail as gracefully as possible.
 130          */
 131         ret = wait_event_interruptible_timeout(error->reset_queue,
 132                                                EXIT_COND,
 133                                                10*HZ);
 134         if (ret == 0) {
 135                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 136                 return -EIO;
 137         } else if (ret < 0) {
 138                 return ret;
 139         }
 140 #undef EXIT_COND
 141
 142         return 0;
 143 }
 144
 145 int i915_mutex_lock_interruptible(struct drm_device *dev)
 146 {
 147         struct drm_i915_private *dev_priv = dev->dev_private;
 148         int ret;
 149
 150         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 151         if (ret)
 152                 return ret;
 153
 154         ret = mutex_lock_interruptible(&dev->struct_mutex);
 155         if (ret)
 156                 return ret;
 157
 158         WARN_ON(i915_verify_lists(dev));
 159         return 0;
 160 }
 161
 162 static inline bool
 163 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 164 {
 165         return i915_gem_obj_bound_any(obj) && !obj->active;
 166 }
 167
 168 int
 169 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 170                     struct drm_file *file)
 171 {
 172         struct drm_i915_private *dev_priv = dev->dev_private;
 173         struct drm_i915_gem_init *args = data;
 174
 175         if (drm_core_check_feature(dev, DRIVER_MODESET))
 176                 return -ENODEV;
 177
 178         if (args->gtt_start >= args->gtt_end ||
 179             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 180                 return -EINVAL;
 181
 182         /* GEM with user mode setting was never supported on ilk and later. */
 183         if (INTEL_INFO(dev)->gen >= 5)
 184                 return -ENODEV;
 185
 186         mutex_lock(&dev->struct_mutex);
 187         i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
 188                                   args->gtt_end);
 189         dev_priv->gtt.mappable_end = args->gtt_end;
 190         mutex_unlock(&dev->struct_mutex);
 191
 192         return 0;
 193 }
 194
 195 int
 196 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 197                             struct drm_file *file)
 198 {
 199         struct drm_i915_private *dev_priv = dev->dev_private;
 200         struct drm_i915_gem_get_aperture *args = data;
 201         struct drm_i915_gem_object *obj;
 202         size_t pinned;
 203
 204         pinned = 0;
 205         mutex_lock(&dev->struct_mutex);
 206         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
 207                 if (obj->pin_count)
 208                         pinned += i915_gem_obj_ggtt_size(obj);
 209         mutex_unlock(&dev->struct_mutex);
 210
 211         args->aper_size = dev_priv->gtt.base.total;
 212         args->aper_available_size = args->aper_size - pinned;
 213
 214         return 0;
 215 }
 216
 217 void *i915_gem_object_alloc(struct drm_device *dev)
 218 {
 219         struct drm_i915_private *dev_priv = dev->dev_private;
 220         return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
 221 }
 222
 223 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 224 {
 225         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 226         kmem_cache_free(dev_priv->slab, obj);
 227 }
 228
 229 static int
 230 i915_gem_create(struct drm_file *file,
 231                 struct drm_device *dev,
 232                 uint64_t size,
 233                 uint32_t *handle_p)
 234 {
 235         struct drm_i915_gem_object *obj;
 236         int ret;
 237         u32 handle;
 238
 239         size = roundup(size, PAGE_SIZE);
 240         if (size == 0)
 241                 return -EINVAL;
 242
 243         /* Allocate the new object */
 244         obj = i915_gem_alloc_object(dev, size);
 245         if (obj == NULL)
 246                 return -ENOMEM;
 247
 248         ret = drm_gem_handle_create(file, &obj->base, &handle);
 249         /* drop reference from allocate - handle holds it now */
 250         drm_gem_object_unreference_unlocked(&obj->base);
 251         if (ret)
 252                 return ret;
 253
 254         *handle_p = handle;
 255         return 0;
 256 }
 257
 258 int
 259 i915_gem_dumb_create(struct drm_file *file,
 260                      struct drm_device *dev,
 261                      struct drm_mode_create_dumb *args)
 262 {
 263         /* have to work out size/pitch and return them */
 264         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 265         args->size = args->pitch * args->height;
 266         return i915_gem_create(file, dev,
 267                                args->size, &args->handle);
 268 }
 269
 270 /**
 271  * Creates a new mm object and returns a handle to it.
 272  */
 273 int
 274 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 275                       struct drm_file *file)
 276 {
 277         struct drm_i915_gem_create *args = data;
 278
 279         return i915_gem_create(file, dev,
 280                                args->size, &args->handle);
 281 }
 282
 283 static inline int
 284 __copy_to_user_swizzled(char __user *cpu_vaddr,
 285                         const char *gpu_vaddr, int gpu_offset,
 286                         int length)
 287 {
 288         int ret, cpu_offset = 0;
 289
 290         while (length > 0) {
 291                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 292                 int this_length = min(cacheline_end - gpu_offset, length);
 293                 int swizzled_gpu_offset = gpu_offset ^ 64;
 294
 295                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 296                                      gpu_vaddr + swizzled_gpu_offset,
 297                                      this_length);
 298                 if (ret)
 299                         return ret + length;
 300
 301                 cpu_offset += this_length;
 302                 gpu_offset += this_length;
 303                 length -= this_length;
 304         }
 305
 306         return 0;
 307 }
 308
 309 static inline int
 310 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 311                           const char __user *cpu_vaddr,
 312                           int length)
 313 {
 314         int ret, cpu_offset = 0;
 315
 316         while (length > 0) {
 317                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 318                 int this_length = min(cacheline_end - gpu_offset, length);
 319                 int swizzled_gpu_offset = gpu_offset ^ 64;
 320
 321                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 322                                        cpu_vaddr + cpu_offset,
 323                                        this_length);
 324                 if (ret)
 325                         return ret + length;
 326
 327                 cpu_offset += this_length;
 328                 gpu_offset += this_length;
 329                 length -= this_length;
 330         }
 331
 332         return 0;
 333 }
 334
 335 /* Per-page copy function for the shmem pread fastpath.
 336  * Flushes invalid cachelines before reading the target if
 337  * needs_clflush is set. */
 338 static int
 339 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 340                  char __user *user_data,
 341                  bool page_do_bit17_swizzling, bool needs_clflush)
 342 {
 343         char *vaddr;
 344         int ret;
 345
 346         if (unlikely(page_do_bit17_swizzling))
 347                 return -EINVAL;
 348
 349         vaddr = kmap_atomic(page);
 350         if (needs_clflush)
 351                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 352                                        page_length);
 353         ret = __copy_to_user_inatomic(user_data,
 354                                       vaddr + shmem_page_offset,
 355                                       page_length);
 356         kunmap_atomic(vaddr);
 357
 358         return ret ? -EFAULT : 0;
 359 }
 360
 361 static void
 362 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 363                              bool swizzled)
 364 {
 365         if (unlikely(swizzled)) {
 366                 unsigned long start = (unsigned long) addr;
 367                 unsigned long end = (unsigned long) addr + length;
 368
 369                 /* For swizzling simply ensure that we always flush both
 370                  * channels. Lame, but simple and it works. Swizzled
 371                  * pwrite/pread is far from a hotpath - current userspace
 372                  * doesn't use it at all. */
 373                 start = round_down(start, 128);
 374                 end = round_up(end, 128);
 375
 376                 drm_clflush_virt_range((void *)start, end - start);
 377         } else {
 378                 drm_clflush_virt_range(addr, length);
 379         }
 380
 381 }
 382
 383 /* Only difference to the fast-path function is that this can handle bit17
 384  * and uses non-atomic copy and kmap functions. */
 385 static int
 386 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 387                  char __user *user_data,
 388                  bool page_do_bit17_swizzling, bool needs_clflush)
 389 {
 390         char *vaddr;
 391         int ret;
 392
 393         vaddr = kmap(page);
 394         if (needs_clflush)
 395                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 396                                              page_length,
 397                                              page_do_bit17_swizzling);
 398
 399         if (page_do_bit17_swizzling)
 400                 ret = __copy_to_user_swizzled(user_data,
 401                                               vaddr, shmem_page_offset,
 402                                               page_length);
 403         else
 404                 ret = __copy_to_user(user_data,
 405                                      vaddr + shmem_page_offset,
 406                                      page_length);
 407         kunmap(page);
 408
 409         return ret ? - EFAULT : 0;
 410 }
 411
 412 static int
 413 i915_gem_shmem_pread(struct drm_device *dev,
 414                      struct drm_i915_gem_object *obj,
 415                      struct drm_i915_gem_pread *args,
 416                      struct drm_file *file)
 417 {
 418         char __user *user_data;
 419         ssize_t remain;
 420         loff_t offset;
 421         int shmem_page_offset, page_length, ret = 0;
 422         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 423         int prefaulted = 0;
 424         int needs_clflush = 0;
 425         struct sg_page_iter sg_iter;
 426
 427         user_data = to_user_ptr(args->data_ptr);
 428         remain = args->size;
 429
 430         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 431
 432         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 433                 /* If we're not in the cpu read domain, set ourself into the gtt
 434                  * read domain and manually flush cachelines (if required). This
 435                  * optimizes for the case when the gpu will dirty the data
 436                  * anyway again before the next pread happens. */
 437                 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
 438                 ret = i915_gem_object_wait_rendering(obj, true);
 439                 if (ret)
 440                         return ret;
 441         }
 442
 443         ret = i915_gem_object_get_pages(obj);
 444         if (ret)
 445                 return ret;
 446
 447         i915_gem_object_pin_pages(obj);
 448
 449         offset = args->offset;
 450
 451         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 452                          offset >> PAGE_SHIFT) {
 453                 struct page *page = sg_page_iter_page(&sg_iter);
 454
 455                 if (remain <= 0)
 456                         break;
 457
 458                 /* Operation in this page
 459                  *
 460                  * shmem_page_offset = offset within page in shmem file
 461                  * page_length = bytes to copy for this page
 462                  */
 463                 shmem_page_offset = offset_in_page(offset);
 464                 page_length = remain;
 465                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 466                         page_length = PAGE_SIZE - shmem_page_offset;
 467
 468                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 469                         (page_to_phys(page) & (1 << 17)) != 0;
 470
 471                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 472                                        user_data, page_do_bit17_swizzling,
 473                                        needs_clflush);
 474                 if (ret == 0)
 475                         goto next_page;
 476
 477                 mutex_unlock(&dev->struct_mutex);
 478
 479                 if (likely(!i915_prefault_disable) && !prefaulted) {
 480                         ret = fault_in_multipages_writeable(user_data, remain);
 481                         /* Userspace is tricking us, but we've already clobbered
 482                          * its pages with the prefault and promised to write the
 483                          * data up to the first fault. Hence ignore any errors
 484                          * and just continue. */
 485                         (void)ret;
 486                         prefaulted = 1;
 487                 }
 488
 489                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 490                                        user_data, page_do_bit17_swizzling,
 491                                        needs_clflush);
 492
 493                 mutex_lock(&dev->struct_mutex);
 494
 495 next_page:
 496                 mark_page_accessed(page);
 497
 498                 if (ret)
 499                         goto out;
 500
 501                 remain -= page_length;
 502                 user_data += page_length;
 503                 offset += page_length;
 504         }
 505
 506 out:
 507         i915_gem_object_unpin_pages(obj);
 508
 509         return ret;
 510 }
 511
 512 /**
 513  * Reads data from the object referenced by handle.
 514  *
 515  * On error, the contents of *data are undefined.
 516  */
 517 int
 518 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 519                      struct drm_file *file)
 520 {
 521         struct drm_i915_gem_pread *args = data;
 522         struct drm_i915_gem_object *obj;
 523         int ret = 0;
 524
 525         if (args->size == 0)
 526                 return 0;
 527
 528         if (!access_ok(VERIFY_WRITE,
 529                        to_user_ptr(args->data_ptr),
 530                        args->size))
 531                 return -EFAULT;
 532
 533         ret = i915_mutex_lock_interruptible(dev);
 534         if (ret)
 535                 return ret;
 536
 537         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 538         if (&obj->base == NULL) {
 539                 ret = -ENOENT;
 540                 goto unlock;
 541         }
 542
 543         /* Bounds check source.  */
 544         if (args->offset > obj->base.size ||
 545             args->size > obj->base.size - args->offset) {
 546                 ret = -EINVAL;
 547                 goto out;
 548         }
 549
 550         /* prime objects have no backing filp to GEM pread/pwrite
 551          * pages from.
 552          */
 553         if (!obj->base.filp) {
 554                 ret = -EINVAL;
 555                 goto out;
 556         }
 557
 558         trace_i915_gem_object_pread(obj, args->offset, args->size);
 559
 560         ret = i915_gem_shmem_pread(dev, obj, args, file);
 561
 562 out:
 563         drm_gem_object_unreference(&obj->base);
 564 unlock:
 565         mutex_unlock(&dev->struct_mutex);
 566         return ret;
 567 }
 568
 569 /* This is the fast write path which cannot handle
 570  * page faults in the source data
 571  */
 572
 573 static inline int
 574 fast_user_write(struct io_mapping *mapping,
 575                 loff_t page_base, int page_offset,
 576                 char __user *user_data,
 577                 int length)
 578 {
 579         void __iomem *vaddr_atomic;
 580         void *vaddr;
 581         unsigned long unwritten;
 582
 583         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 584         /* We can use the cpu mem copy function because this is X86. */
 585         vaddr = (void __force*)vaddr_atomic + page_offset;
 586         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 587                                                       user_data, length);
 588         io_mapping_unmap_atomic(vaddr_atomic);
 589         return unwritten;
 590 }
 591
 592 /**
 593  * This is the fast pwrite path, where we copy the data directly from the
 594  * user into the GTT, uncached.
 595  */
 596 static int
 597 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 598                          struct drm_i915_gem_object *obj,
 599                          struct drm_i915_gem_pwrite *args,
 600                          struct drm_file *file)
 601 {
 602         drm_i915_private_t *dev_priv = dev->dev_private;
 603         ssize_t remain;
 604         loff_t offset, page_base;
 605         char __user *user_data;
 606         int page_offset, page_length, ret;
 607
 608         ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
 609         if (ret)
 610                 goto out;
 611
 612         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 613         if (ret)
 614                 goto out_unpin;
 615
 616         ret = i915_gem_object_put_fence(obj);
 617         if (ret)
 618                 goto out_unpin;
 619
 620         user_data = to_user_ptr(args->data_ptr);
 621         remain = args->size;
 622
 623         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 624
 625         while (remain > 0) {
 626                 /* Operation in this page
 627                  *
 628                  * page_base = page offset within aperture
 629                  * page_offset = offset within page
 630                  * page_length = bytes to copy for this page
 631                  */
 632                 page_base = offset & PAGE_MASK;
 633                 page_offset = offset_in_page(offset);
 634                 page_length = remain;
 635                 if ((page_offset + remain) > PAGE_SIZE)
 636                         page_length = PAGE_SIZE - page_offset;
 637
 638                 /* If we get a fault while copying data, then (presumably) our
 639                  * source page isn't available.  Return the error and we'll
 640                  * retry in the slow path.
 641                  */
 642                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
 643                                     page_offset, user_data, page_length)) {
 644                         ret = -EFAULT;
 645                         goto out_unpin;
 646                 }
 647
 648                 remain -= page_length;
 649                 user_data += page_length;
 650                 offset += page_length;
 651         }
 652
 653 out_unpin:
 654         i915_gem_object_unpin(obj);
 655 out:
 656         return ret;
 657 }
 658
 659 /* Per-page copy function for the shmem pwrite fastpath.
 660  * Flushes invalid cachelines before writing to the target if
 661  * needs_clflush_before is set and flushes out any written cachelines after
 662  * writing if needs_clflush is set. */
 663 static int
 664 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 665                   char __user *user_data,
 666                   bool page_do_bit17_swizzling,
 667                   bool needs_clflush_before,
 668                   bool needs_clflush_after)
 669 {
 670         char *vaddr;
 671         int ret;
 672
 673         if (unlikely(page_do_bit17_swizzling))
 674                 return -EINVAL;
 675
 676         vaddr = kmap_atomic(page);
 677         if (needs_clflush_before)
 678                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 679                                        page_length);
 680         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 681                                                 user_data,
 682                                                 page_length);
 683         if (needs_clflush_after)
 684                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 685                                        page_length);
 686         kunmap_atomic(vaddr);
 687
 688         return ret ? -EFAULT : 0;
 689 }
 690
 691 /* Only difference to the fast-path function is that this can handle bit17
 692  * and uses non-atomic copy and kmap functions. */
 693 static int
 694 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 695                   char __user *user_data,
 696                   bool page_do_bit17_swizzling,
 697                   bool needs_clflush_before,
 698                   bool needs_clflush_after)
 699 {
 700         char *vaddr;
 701         int ret;
 702
 703         vaddr = kmap(page);
 704         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 705                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 706                                              page_length,
 707                                              page_do_bit17_swizzling);
 708         if (page_do_bit17_swizzling)
 709                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 710                                                 user_data,
 711                                                 page_length);
 712         else
 713                 ret = __copy_from_user(vaddr + shmem_page_offset,
 714                                        user_data,
 715                                        page_length);
 716         if (needs_clflush_after)
 717                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 718                                              page_length,
 719                                              page_do_bit17_swizzling);
 720         kunmap(page);
 721
 722         return ret ? -EFAULT : 0;
 723 }
 724
 725 static int
 726 i915_gem_shmem_pwrite(struct drm_device *dev,
 727                       struct drm_i915_gem_object *obj,
 728                       struct drm_i915_gem_pwrite *args,
 729                       struct drm_file *file)
 730 {
 731         ssize_t remain;
 732         loff_t offset;
 733         char __user *user_data;
 734         int shmem_page_offset, page_length, ret = 0;
 735         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 736         int hit_slowpath = 0;
 737         int needs_clflush_after = 0;
 738         int needs_clflush_before = 0;
 739         struct sg_page_iter sg_iter;
 740
 741         user_data = to_user_ptr(args->data_ptr);
 742         remain = args->size;
 743
 744         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 745
 746         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 747                 /* If we're not in the cpu write domain, set ourself into the gtt
 748                  * write domain and manually flush cachelines (if required). This
 749                  * optimizes for the case when the gpu will use the data
 750                  * right away and we therefore have to clflush anyway. */
 751                 needs_clflush_after = cpu_write_needs_clflush(obj);
 752                 ret = i915_gem_object_wait_rendering(obj, false);
 753                 if (ret)
 754                         return ret;
 755         }
 756         /* Same trick applies to invalidate partially written cachelines read
 757          * before writing. */
 758         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
 759                 needs_clflush_before =
 760                         !cpu_cache_is_coherent(dev, obj->cache_level);
 761
 762         ret = i915_gem_object_get_pages(obj);
 763         if (ret)
 764                 return ret;
 765
 766         i915_gem_object_pin_pages(obj);
 767
 768         offset = args->offset;
 769         obj->dirty = 1;
 770
 771         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 772                          offset >> PAGE_SHIFT) {
 773                 struct page *page = sg_page_iter_page(&sg_iter);
 774                 int partial_cacheline_write;
 775
 776                 if (remain <= 0)
 777                         break;
 778
 779                 /* Operation in this page
 780                  *
 781                  * shmem_page_offset = offset within page in shmem file
 782                  * page_length = bytes to copy for this page
 783                  */
 784                 shmem_page_offset = offset_in_page(offset);
 785
 786                 page_length = remain;
 787                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 788                         page_length = PAGE_SIZE - shmem_page_offset;
 789
 790                 /* If we don't overwrite a cacheline completely we need to be
 791                  * careful to have up-to-date data by first clflushing. Don't
 792                  * overcomplicate things and flush the entire patch. */
 793                 partial_cacheline_write = needs_clflush_before &&
 794                         ((shmem_page_offset | page_length)
 795                                 & (boot_cpu_data.x86_clflush_size - 1));
 796
 797                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 798                         (page_to_phys(page) & (1 << 17)) != 0;
 799
 800                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 801                                         user_data, page_do_bit17_swizzling,
 802                                         partial_cacheline_write,
 803                                         needs_clflush_after);
 804                 if (ret == 0)
 805                         goto next_page;
 806
 807                 hit_slowpath = 1;
 808                 mutex_unlock(&dev->struct_mutex);
 809                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 810                                         user_data, page_do_bit17_swizzling,
 811                                         partial_cacheline_write,
 812                                         needs_clflush_after);
 813
 814                 mutex_lock(&dev->struct_mutex);
 815
 816 next_page:
 817                 set_page_dirty(page);
 818                 mark_page_accessed(page);
 819
 820                 if (ret)
 821                         goto out;
 822
 823                 remain -= page_length;
 824                 user_data += page_length;
 825                 offset += page_length;
 826         }
 827
 828 out:
 829         i915_gem_object_unpin_pages(obj);
 830
 831         if (hit_slowpath) {
 832                 /*
 833                  * Fixup: Flush cpu caches in case we didn't flush the dirty
 834                  * cachelines in-line while writing and the object moved
 835                  * out of the cpu write domain while we've dropped the lock.
 836                  */
 837                 if (!needs_clflush_after &&
 838                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 839                         if (i915_gem_clflush_object(obj, obj->pin_display))
 840                                 i915_gem_chipset_flush(dev);
 841                 }
 842         }
 843
 844         if (needs_clflush_after)
 845                 i915_gem_chipset_flush(dev);
 846
 847         return ret;
 848 }
 849
 850 /**
 851  * Writes data to the object referenced by handle.
 852  *
 853  * On error, the contents of the buffer that were to be modified are undefined.
 854  */
 855 int
 856 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 857                       struct drm_file *file)
 858 {
 859         struct drm_i915_gem_pwrite *args = data;
 860         struct drm_i915_gem_object *obj;
 861         int ret;
 862
 863         if (args->size == 0)
 864                 return 0;
 865
 866         if (!access_ok(VERIFY_READ,
 867                        to_user_ptr(args->data_ptr),
 868                        args->size))
 869                 return -EFAULT;
 870
 871         if (likely(!i915_prefault_disable)) {
 872                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
 873                                                    args->size);
 874                 if (ret)
 875                         return -EFAULT;
 876         }
 877
 878         ret = i915_mutex_lock_interruptible(dev);
 879         if (ret)
 880                 return ret;
 881
 882         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 883         if (&obj->base == NULL) {
 884                 ret = -ENOENT;
 885                 goto unlock;
 886         }
 887
 888         /* Bounds check destination. */
 889         if (args->offset > obj->base.size ||
 890             args->size > obj->base.size - args->offset) {
 891                 ret = -EINVAL;
 892                 goto out;
 893         }
 894
 895         /* prime objects have no backing filp to GEM pread/pwrite
 896          * pages from.
 897          */
 898         if (!obj->base.filp) {
 899                 ret = -EINVAL;
 900                 goto out;
 901         }
 902
 903         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 904
 905         ret = -EFAULT;
 906         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 907          * it would end up going through the fenced access, and we'll get
 908          * different detiling behavior between reading and writing.
 909          * pread/pwrite currently are reading and writing from the CPU
 910          * perspective, requiring manual detiling by the client.
 911          */
 912         if (obj->phys_obj) {
 913                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 914                 goto out;
 915         }
 916
 917         if (obj->tiling_mode == I915_TILING_NONE &&
 918             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
 919             cpu_write_needs_clflush(obj)) {
 920                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 921                 /* Note that the gtt paths might fail with non-page-backed user
 922                  * pointers (e.g. gtt mappings when moving data between
 923                  * textures). Fallback to the shmem path in that case. */
 924         }
 925
 926         if (ret == -EFAULT || ret == -ENOSPC)
 927                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 928
 929 out:
 930         drm_gem_object_unreference(&obj->base);
 931 unlock:
 932         mutex_unlock(&dev->struct_mutex);
 933         return ret;
 934 }
 935
 936 int
 937 i915_gem_check_wedge(struct i915_gpu_error *error,
 938                      bool interruptible)
 939 {
 940         if (i915_reset_in_progress(error)) {
 941                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 942                  * -EIO unconditionally for these. */
 943                 if (!interruptible)
 944                         return -EIO;
 945
 946                 /* Recovery complete, but the reset failed ... */
 947                 if (i915_terminally_wedged(error))
 948                         return -EIO;
 949
 950                 return -EAGAIN;
 951         }
 952
 953         return 0;
 954 }
 955
 956 /*
 957  * Compare seqno against outstanding lazy request. Emit a request if they are
 958  * equal.
 959  */
 960 static int
 961 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
 962 {
 963         int ret;
 964
 965         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 966
 967         ret = 0;
 968         if (seqno == ring->outstanding_lazy_seqno)
 969                 ret = i915_add_request(ring, NULL);
 970
 971         return ret;
 972 }
 973
 974 static void fake_irq(unsigned long data)
 975 {
 976         wake_up_process((struct task_struct *)data);
 977 }
 978
 979 static bool missed_irq(struct drm_i915_private *dev_priv,
 980                        struct intel_ring_buffer *ring)
 981 {
 982         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
 983 }
 984
 985 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
 986 {
 987         if (file_priv == NULL)
 988                 return true;
 989
 990         return !atomic_xchg(&file_priv->rps_wait_boost, true);
 991 }
 992
 993 /**
 994  * __wait_seqno - wait until execution of seqno has finished
 995  * @ring: the ring expected to report seqno
 996  * @seqno: duh!
 997  * @reset_counter: reset sequence associated with the given seqno
 998  * @interruptible: do an interruptible wait (normally yes)
 999  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1000  *
1001  * Note: It is of utmost importance that the passed in seqno and reset_counter
1002  * values have been read by the caller in an smp safe manner. Where read-side
1003  * locks are involved, it is sufficient to read the reset_counter before
1004  * unlocking the lock that protects the seqno. For lockless tricks, the
1005  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1006  * inserted.
1007  *
1008  * Returns 0 if the seqno was found within the alloted time. Else returns the
1009  * errno with remaining time filled in timeout argument.
1010  */
1011 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1012                         unsigned reset_counter,
1013                         bool interruptible,
1014                         struct timespec *timeout,
1015                         struct drm_i915_file_private *file_priv)
1016 {
1017         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1018         struct timespec before, now;
1019         DEFINE_WAIT(wait);
1020         long timeout_jiffies;
1021         int ret;
1022
1023         WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
1024
1025         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
1026                 return 0;
1027
1028         timeout_jiffies = timeout ? timespec_to_jiffies_timeout(timeout) : 1;
1029
1030         if (dev_priv->info->gen >= 6 && can_wait_boost(file_priv)) {
1031                 gen6_rps_boost(dev_priv);
1032                 if (file_priv)
1033                         mod_delayed_work(dev_priv->wq,
1034                                          &file_priv->mm.idle_work,
1035                                          msecs_to_jiffies(100));
1036         }
1037
1038         if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)) &&
1039             WARN_ON(!ring->irq_get(ring)))
1040                 return -ENODEV;
1041
1042         /* Record current time in case interrupted by signal, or wedged */
1043         trace_i915_gem_request_wait_begin(ring, seqno);
1044         getrawmonotonic(&before);
1045         for (;;) {
1046                 struct timer_list timer;
1047                 unsigned long expire;
1048
1049                 prepare_to_wait(&ring->irq_queue, &wait,
1050                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1051
1052                 /* We need to check whether any gpu reset happened in between
1053                  * the caller grabbing the seqno and now ... */
1054                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1055                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1056                          * is truely gone. */
1057                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1058                         if (ret == 0)
1059                                 ret = -EAGAIN;
1060                         break;
1061                 }
1062
1063                 if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
1064                         ret = 0;
1065                         break;
1066                 }
1067
1068                 if (interruptible && signal_pending(current)) {
1069                         ret = -ERESTARTSYS;
1070                         break;
1071                 }
1072
1073                 if (timeout_jiffies <= 0) {
1074                         ret = -ETIME;
1075                         break;
1076                 }
1077
1078                 timer.function = NULL;
1079                 if (timeout || missed_irq(dev_priv, ring)) {
1080                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1081                         expire = jiffies + (missed_irq(dev_priv, ring) ? 1: timeout_jiffies);
1082                         mod_timer(&timer, expire);
1083                 }
1084
1085                 io_schedule();
1086
1087                 if (timeout)
1088                         timeout_jiffies = expire - jiffies;
1089
1090                 if (timer.function) {
1091                         del_singleshot_timer_sync(&timer);
1092                         destroy_timer_on_stack(&timer);
1093                 }
1094         }
1095         getrawmonotonic(&now);
1096         trace_i915_gem_request_wait_end(ring, seqno);
1097
1098         ring->irq_put(ring);
1099
1100         finish_wait(&ring->irq_queue, &wait);
1101
1102         if (timeout) {
1103                 struct timespec sleep_time = timespec_sub(now, before);
1104                 *timeout = timespec_sub(*timeout, sleep_time);
1105                 if (!timespec_valid(timeout)) /* i.e. negative time remains */
1106                         set_normalized_timespec(timeout, 0, 0);
1107         }
1108
1109         return ret;
1110 }
1111
1112 /**
1113  * Waits for a sequence number to be signaled, and cleans up the
1114  * request and object lists appropriately for that event.
1115  */
1116 int
1117 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1118 {
1119         struct drm_device *dev = ring->dev;
1120         struct drm_i915_private *dev_priv = dev->dev_private;
1121         bool interruptible = dev_priv->mm.interruptible;
1122         int ret;
1123
1124         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1125         BUG_ON(seqno == 0);
1126
1127         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1128         if (ret)
1129                 return ret;
1130
1131         ret = i915_gem_check_olr(ring, seqno);
1132         if (ret)
1133                 return ret;
1134
1135         return __wait_seqno(ring, seqno,
1136                             atomic_read(&dev_priv->gpu_error.reset_counter),
1137                             interruptible, NULL, NULL);
1138 }
1139
1140 static int
1141 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
1142                                      struct intel_ring_buffer *ring)
1143 {
1144         i915_gem_retire_requests_ring(ring);
1145
1146         /* Manually manage the write flush as we may have not yet
1147          * retired the buffer.
1148          *
1149          * Note that the last_write_seqno is always the earlier of
1150          * the two (read/write) seqno, so if we haved successfully waited,
1151          * we know we have passed the last write.
1152          */
1153         obj->last_write_seqno = 0;
1154         obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1155
1156         return 0;
1157 }
1158
1159 /**
1160  * Ensures that all rendering to the object has completed and the object is
1161  * safe to unbind from the GTT or access from the CPU.
1162  */
1163 static __must_check int
1164 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1165                                bool readonly)
1166 {
1167         struct intel_ring_buffer *ring = obj->ring;
1168         u32 seqno;
1169         int ret;
1170
1171         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1172         if (seqno == 0)
1173                 return 0;
1174
1175         ret = i915_wait_seqno(ring, seqno);
1176         if (ret)
1177                 return ret;
1178
1179         return i915_gem_object_wait_rendering__tail(obj, ring);
1180 }
1181
1182 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1183  * as the object state may change during this call.
1184  */
1185 static __must_check int
1186 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1187                                             struct drm_file *file,
1188                                             bool readonly)
1189 {
1190         struct drm_device *dev = obj->base.dev;
1191         struct drm_i915_private *dev_priv = dev->dev_private;
1192         struct intel_ring_buffer *ring = obj->ring;
1193         unsigned reset_counter;
1194         u32 seqno;
1195         int ret;
1196
1197         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1198         BUG_ON(!dev_priv->mm.interruptible);
1199
1200         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1201         if (seqno == 0)
1202                 return 0;
1203
1204         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1205         if (ret)
1206                 return ret;
1207
1208         ret = i915_gem_check_olr(ring, seqno);
1209         if (ret)
1210                 return ret;
1211
1212         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1213         mutex_unlock(&dev->struct_mutex);
1214         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file->driver_priv);
1215         mutex_lock(&dev->struct_mutex);
1216         if (ret)
1217                 return ret;
1218
1219         return i915_gem_object_wait_rendering__tail(obj, ring);
1220 }
1221
1222 /**
1223  * Called when user space prepares to use an object with the CPU, either
1224  * through the mmap ioctl's mapping or a GTT mapping.
1225  */
1226 int
1227 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1228                           struct drm_file *file)
1229 {
1230         struct drm_i915_gem_set_domain *args = data;
1231         struct drm_i915_gem_object *obj;
1232         uint32_t read_domains = args->read_domains;
1233         uint32_t write_domain = args->write_domain;
1234         int ret;
1235
1236         /* Only handle setting domains to types used by the CPU. */
1237         if (write_domain & I915_GEM_GPU_DOMAINS)
1238                 return -EINVAL;
1239
1240         if (read_domains & I915_GEM_GPU_DOMAINS)
1241                 return -EINVAL;
1242
1243         /* Having something in the write domain implies it's in the read
1244          * domain, and only that read domain.  Enforce that in the request.
1245          */
1246         if (write_domain != 0 && read_domains != write_domain)
1247                 return -EINVAL;
1248
1249         ret = i915_mutex_lock_interruptible(dev);
1250         if (ret)
1251                 return ret;
1252
1253         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1254         if (&obj->base == NULL) {
1255                 ret = -ENOENT;
1256                 goto unlock;
1257         }
1258
1259         /* Try to flush the object off the GPU without holding the lock.
1260          * We will repeat the flush holding the lock in the normal manner
1261          * to catch cases where we are gazumped.
1262          */
1263         ret = i915_gem_object_wait_rendering__nonblocking(obj, file, !write_domain);
1264         if (ret)
1265                 goto unref;
1266
1267         if (read_domains & I915_GEM_DOMAIN_GTT) {
1268                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1269
1270                 /* Silently promote "you're not bound, there was nothing to do"
1271                  * to success, since the client was just asking us to
1272                  * make sure everything was done.
1273                  */
1274                 if (ret == -EINVAL)
1275                         ret = 0;
1276         } else {
1277                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1278         }
1279
1280 unref:
1281         drm_gem_object_unreference(&obj->base);
1282 unlock:
1283         mutex_unlock(&dev->struct_mutex);
1284         return ret;
1285 }
1286
1287 /**
1288  * Called when user space has done writes to this buffer
1289  */
1290 int
1291 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1292                          struct drm_file *file)
1293 {
1294         struct drm_i915_gem_sw_finish *args = data;
1295         struct drm_i915_gem_object *obj;
1296         int ret = 0;
1297
1298         ret = i915_mutex_lock_interruptible(dev);
1299         if (ret)
1300                 return ret;
1301
1302         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1303         if (&obj->base == NULL) {
1304                 ret = -ENOENT;
1305                 goto unlock;
1306         }
1307
1308         /* Pinned buffers may be scanout, so flush the cache */
1309         if (obj->pin_display)
1310                 i915_gem_object_flush_cpu_write_domain(obj, true);
1311
1312         drm_gem_object_unreference(&obj->base);
1313 unlock:
1314         mutex_unlock(&dev->struct_mutex);
1315         return ret;
1316 }
1317
1318 /**
1319  * Maps the contents of an object, returning the address it is mapped
1320  * into.
1321  *
1322  * While the mapping holds a reference on the contents of the object, it doesn't
1323  * imply a ref on the object itself.
1324  */
1325 int
1326 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1327                     struct drm_file *file)
1328 {
1329         struct drm_i915_gem_mmap *args = data;
1330         struct drm_gem_object *obj;
1331         unsigned long addr;
1332
1333         obj = drm_gem_object_lookup(dev, file, args->handle);
1334         if (obj == NULL)
1335                 return -ENOENT;
1336
1337         /* prime objects have no backing filp to GEM mmap
1338          * pages from.
1339          */
1340         if (!obj->filp) {
1341                 drm_gem_object_unreference_unlocked(obj);
1342                 return -EINVAL;
1343         }
1344
1345         addr = vm_mmap(obj->filp, 0, args->size,
1346                        PROT_READ | PROT_WRITE, MAP_SHARED,
1347                        args->offset);
1348         drm_gem_object_unreference_unlocked(obj);
1349         if (IS_ERR((void *)addr))
1350                 return addr;
1351
1352         args->addr_ptr = (uint64_t) addr;
1353
1354         return 0;
1355 }
1356
1357 /**
1358  * i915_gem_fault - fault a page into the GTT
1359  * vma: VMA in question
1360  * vmf: fault info
1361  *
1362  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1363  * from userspace.  The fault handler takes care of binding the object to
1364  * the GTT (if needed), allocating and programming a fence register (again,
1365  * only if needed based on whether the old reg is still valid or the object
1366  * is tiled) and inserting a new PTE into the faulting process.
1367  *
1368  * Note that the faulting process may involve evicting existing objects
1369  * from the GTT and/or fence registers to make room.  So performance may
1370  * suffer if the GTT working set is large or there are few fence registers
1371  * left.
1372  */
1373 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1374 {
1375         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1376         struct drm_device *dev = obj->base.dev;
1377         drm_i915_private_t *dev_priv = dev->dev_private;
1378         pgoff_t page_offset;
1379         unsigned long pfn;
1380         int ret = 0;
1381         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1382
1383         /* We don't use vmf->pgoff since that has the fake offset */
1384         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1385                 PAGE_SHIFT;
1386
1387         ret = i915_mutex_lock_interruptible(dev);
1388         if (ret)
1389                 goto out;
1390
1391         trace_i915_gem_object_fault(obj, page_offset, true, write);
1392
1393         /* Access to snoopable pages through the GTT is incoherent. */
1394         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1395                 ret = -EINVAL;
1396                 goto unlock;
1397         }
1398
1399         /* Now bind it into the GTT if needed */
1400         ret = i915_gem_obj_ggtt_pin(obj,  0, true, false);
1401         if (ret)
1402                 goto unlock;
1403
1404         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1405         if (ret)
1406                 goto unpin;
1407
1408         ret = i915_gem_object_get_fence(obj);
1409         if (ret)
1410                 goto unpin;
1411
1412         obj->fault_mappable = true;
1413
1414         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1415         pfn >>= PAGE_SHIFT;
1416         pfn += page_offset;
1417
1418         /* Finally, remap it using the new GTT offset */
1419         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1420 unpin:
1421         i915_gem_object_unpin(obj);
1422 unlock:
1423         mutex_unlock(&dev->struct_mutex);
1424 out:
1425         switch (ret) {
1426         case -EIO:
1427                 /* If this -EIO is due to a gpu hang, give the reset code a
1428                  * chance to clean up the mess. Otherwise return the proper
1429                  * SIGBUS. */
1430                 if (i915_terminally_wedged(&dev_priv->gpu_error))
1431                         return VM_FAULT_SIGBUS;
1432         case -EAGAIN:
1433                 /*
1434                  * EAGAIN means the gpu is hung and we'll wait for the error
1435                  * handler to reset everything when re-faulting in
1436                  * i915_mutex_lock_interruptible.
1437                  */
1438         case 0:
1439         case -ERESTARTSYS:
1440         case -EINTR:
1441         case -EBUSY:
1442                 /*
1443                  * EBUSY is ok: this just means that another thread
1444                  * already did the job.
1445                  */
1446                 return VM_FAULT_NOPAGE;
1447         case -ENOMEM:
1448                 return VM_FAULT_OOM;
1449         case -ENOSPC:
1450                 return VM_FAULT_SIGBUS;
1451         default:
1452                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1453                 return VM_FAULT_SIGBUS;
1454         }
1455 }
1456
1457 /**
1458  * i915_gem_release_mmap - remove physical page mappings
1459  * @obj: obj in question
1460  *
1461  * Preserve the reservation of the mmapping with the DRM core code, but
1462  * relinquish ownership of the pages back to the system.
1463  *
1464  * It is vital that we remove the page mapping if we have mapped a tiled
1465  * object through the GTT and then lose the fence register due to
1466  * resource pressure. Similarly if the object has been moved out of the
1467  * aperture, than pages mapped into userspace must be revoked. Removing the
1468  * mapping will then trigger a page fault on the next user access, allowing
1469  * fixup by i915_gem_fault().
1470  */
1471 void
1472 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1473 {
1474         if (!obj->fault_mappable)
1475                 return;
1476
1477         drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
1478         obj->fault_mappable = false;
1479 }
1480
1481 uint32_t
1482 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1483 {
1484         uint32_t gtt_size;
1485
1486         if (INTEL_INFO(dev)->gen >= 4 ||
1487             tiling_mode == I915_TILING_NONE)
1488                 return size;
1489
1490         /* Previous chips need a power-of-two fence region when tiling */
1491         if (INTEL_INFO(dev)->gen == 3)
1492                 gtt_size = 1024*1024;
1493         else
1494                 gtt_size = 512*1024;
1495
1496         while (gtt_size < size)
1497                 gtt_size <<= 1;
1498
1499         return gtt_size;
1500 }
1501
1502 /**
1503  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1504  * @obj: object to check
1505  *
1506  * Return the required GTT alignment for an object, taking into account
1507  * potential fence register mapping.
1508  */
1509 uint32_t
1510 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1511                            int tiling_mode, bool fenced)
1512 {
1513         /*
1514          * Minimum alignment is 4k (GTT page size), but might be greater
1515          * if a fence register is needed for the object.
1516          */
1517         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1518             tiling_mode == I915_TILING_NONE)
1519                 return 4096;
1520
1521         /*
1522          * Previous chips need to be aligned to the size of the smallest
1523          * fence register that can contain the object.
1524          */
1525         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1526 }
1527
1528 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1529 {
1530         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1531         int ret;
1532
1533         if (drm_vma_node_has_offset(&obj->base.vma_node))
1534                 return 0;
1535
1536         dev_priv->mm.shrinker_no_lock_stealing = true;
1537
1538         ret = drm_gem_create_mmap_offset(&obj->base);
1539         if (ret != -ENOSPC)
1540                 goto out;
1541
1542         /* Badly fragmented mmap space? The only way we can recover
1543          * space is by destroying unwanted objects. We can't randomly release
1544          * mmap_offsets as userspace expects them to be persistent for the
1545          * lifetime of the objects. The closest we can is to release the
1546          * offsets on purgeable objects by truncating it and marking it purged,
1547          * which prevents userspace from ever using that object again.
1548          */
1549         i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1550         ret = drm_gem_create_mmap_offset(&obj->base);
1551         if (ret != -ENOSPC)
1552                 goto out;
1553
1554         i915_gem_shrink_all(dev_priv);
1555         ret = drm_gem_create_mmap_offset(&obj->base);
1556 out:
1557         dev_priv->mm.shrinker_no_lock_stealing = false;
1558
1559         return ret;
1560 }
1561
1562 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1563 {
1564         drm_gem_free_mmap_offset(&obj->base);
1565 }
1566
1567 int
1568 i915_gem_mmap_gtt(struct drm_file *file,
1569                   struct drm_device *dev,
1570                   uint32_t handle,
1571                   uint64_t *offset)
1572 {
1573         struct drm_i915_private *dev_priv = dev->dev_private;
1574         struct drm_i915_gem_object *obj;
1575         int ret;
1576
1577         ret = i915_mutex_lock_interruptible(dev);
1578         if (ret)
1579                 return ret;
1580
1581         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1582         if (&obj->base == NULL) {
1583                 ret = -ENOENT;
1584                 goto unlock;
1585         }
1586
1587         if (obj->base.size > dev_priv->gtt.mappable_end) {
1588                 ret = -E2BIG;
1589                 goto out;
1590         }
1591
1592         if (obj->madv != I915_MADV_WILLNEED) {
1593                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1594                 ret = -EINVAL;
1595                 goto out;
1596         }
1597
1598         ret = i915_gem_object_create_mmap_offset(obj);
1599         if (ret)
1600                 goto out;
1601
1602         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1603
1604 out:
1605         drm_gem_object_unreference(&obj->base);
1606 unlock:
1607         mutex_unlock(&dev->struct_mutex);
1608         return ret;
1609 }
1610
1611 /**
1612  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1613  * @dev: DRM device
1614  * @data: GTT mapping ioctl data
1615  * @file: GEM object info
1616  *
1617  * Simply returns the fake offset to userspace so it can mmap it.
1618  * The mmap call will end up in drm_gem_mmap(), which will set things
1619  * up so we can get faults in the handler above.
1620  *
1621  * The fault handler will take care of binding the object into the GTT
1622  * (since it may have been evicted to make room for something), allocating
1623  * a fence register, and mapping the appropriate aperture address into
1624  * userspace.
1625  */
1626 int
1627 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1628                         struct drm_file *file)
1629 {
1630         struct drm_i915_gem_mmap_gtt *args = data;
1631
1632         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1633 }
1634
1635 /* Immediately discard the backing storage */
1636 static void
1637 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1638 {
1639         struct inode *inode;
1640
1641         i915_gem_object_free_mmap_offset(obj);
1642
1643         if (obj->base.filp == NULL)
1644                 return;
1645
1646         /* Our goal here is to return as much of the memory as
1647          * is possible back to the system as we are called from OOM.
1648          * To do this we must instruct the shmfs to drop all of its
1649          * backing pages, *now*.
1650          */
1651         inode = file_inode(obj->base.filp);
1652         shmem_truncate_range(inode, 0, (loff_t)-1);
1653
1654         obj->madv = __I915_MADV_PURGED;
1655 }
1656
1657 static inline int
1658 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1659 {
1660         return obj->madv == I915_MADV_DONTNEED;
1661 }
1662
1663 static void
1664 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1665 {
1666         struct sg_page_iter sg_iter;
1667         int ret;
1668
1669         BUG_ON(obj->madv == __I915_MADV_PURGED);
1670
1671         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1672         if (ret) {
1673                 /* In the event of a disaster, abandon all caches and
1674                  * hope for the best.
1675                  */
1676                 WARN_ON(ret != -EIO);
1677                 i915_gem_clflush_object(obj, true);
1678                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1679         }
1680
1681         if (i915_gem_object_needs_bit17_swizzle(obj))
1682                 i915_gem_object_save_bit_17_swizzle(obj);
1683
1684         if (obj->madv == I915_MADV_DONTNEED)
1685                 obj->dirty = 0;
1686
1687         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1688                 struct page *page = sg_page_iter_page(&sg_iter);
1689
1690                 if (obj->dirty)
1691                         set_page_dirty(page);
1692
1693                 if (obj->madv == I915_MADV_WILLNEED)
1694                         mark_page_accessed(page);
1695
1696                 page_cache_release(page);
1697         }
1698         obj->dirty = 0;
1699
1700         sg_free_table(obj->pages);
1701         kfree(obj->pages);
1702 }
1703
1704 int
1705 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1706 {
1707         const struct drm_i915_gem_object_ops *ops = obj->ops;
1708
1709         if (obj->pages == NULL)
1710                 return 0;
1711
1712         if (obj->pages_pin_count)
1713                 return -EBUSY;
1714
1715         BUG_ON(i915_gem_obj_bound_any(obj));
1716
1717         /* ->put_pages might need to allocate memory for the bit17 swizzle
1718          * array, hence protect them from being reaped by removing them from gtt
1719          * lists early. */
1720         list_del(&obj->global_list);
1721
1722         ops->put_pages(obj);
1723         obj->pages = NULL;
1724
1725         if (i915_gem_object_is_purgeable(obj))
1726                 i915_gem_object_truncate(obj);
1727
1728         return 0;
1729 }
1730
1731 static unsigned long
1732 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
1733                   bool purgeable_only)
1734 {
1735         struct list_head still_bound_list;
1736         struct drm_i915_gem_object *obj, *next;
1737         unsigned long count = 0;
1738
1739         list_for_each_entry_safe(obj, next,
1740                                  &dev_priv->mm.unbound_list,
1741                                  global_list) {
1742                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1743                     i915_gem_object_put_pages(obj) == 0) {
1744                         count += obj->base.size >> PAGE_SHIFT;
1745                         if (count >= target)
1746                                 return count;
1747                 }
1748         }
1749
1750         /*
1751          * As we may completely rewrite the bound list whilst unbinding
1752          * (due to retiring requests) we have to strictly process only
1753          * one element of the list at the time, and recheck the list
1754          * on every iteration.
1755          */
1756         INIT_LIST_HEAD(&still_bound_list);
1757         while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
1758                 struct i915_vma *vma, *v;
1759
1760                 obj = list_first_entry(&dev_priv->mm.bound_list,
1761                                        typeof(*obj), global_list);
1762                 list_move_tail(&obj->global_list, &still_bound_list);
1763
1764                 if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
1765                         continue;
1766
1767                 /*
1768                  * Hold a reference whilst we unbind this object, as we may
1769                  * end up waiting for and retiring requests. This might
1770                  * release the final reference (held by the active list)
1771                  * and result in the object being freed from under us.
1772                  * in this object being freed.
1773                  *
1774                  * Note 1: Shrinking the bound list is special since only active
1775                  * (and hence bound objects) can contain such limbo objects, so
1776                  * we don't need special tricks for shrinking the unbound list.
1777                  * The only other place where we have to be careful with active
1778                  * objects suddenly disappearing due to retiring requests is the
1779                  * eviction code.
1780                  *
1781                  * Note 2: Even though the bound list doesn't hold a reference
1782                  * to the object we can safely grab one here: The final object
1783                  * unreferencing and the bound_list are both protected by the
1784                  * dev->struct_mutex and so we won't ever be able to observe an
1785                  * object on the bound_list with a reference count equals 0.
1786                  */
1787                 drm_gem_object_reference(&obj->base);
1788
1789                 list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
1790                         if (i915_vma_unbind(vma))
1791                                 break;
1792
1793                 if (i915_gem_object_put_pages(obj) == 0)
1794                         count += obj->base.size >> PAGE_SHIFT;
1795
1796                 drm_gem_object_unreference(&obj->base);
1797         }
1798         list_splice(&still_bound_list, &dev_priv->mm.bound_list);
1799
1800         return count;
1801 }
1802
1803 static unsigned long
1804 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1805 {
1806         return __i915_gem_shrink(dev_priv, target, true);
1807 }
1808
1809 static unsigned long
1810 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1811 {
1812         struct drm_i915_gem_object *obj, *next;
1813         long freed = 0;
1814
1815         i915_gem_evict_everything(dev_priv->dev);
1816
1817         list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
1818                                  global_list) {
1819                 if (i915_gem_object_put_pages(obj) == 0)
1820                         freed += obj->base.size >> PAGE_SHIFT;
1821         }
1822         return freed;
1823 }
1824
1825 static int
1826 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1827 {
1828         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1829         int page_count, i;
1830         struct address_space *mapping;
1831         struct sg_table *st;
1832         struct scatterlist *sg;
1833         struct sg_page_iter sg_iter;
1834         struct page *page;
1835         unsigned long last_pfn = 0;     /* suppress gcc warning */
1836         gfp_t gfp;
1837
1838         /* Assert that the object is not currently in any GPU domain. As it
1839          * wasn't in the GTT, there shouldn't be any way it could have been in
1840          * a GPU cache
1841          */
1842         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1843         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1844
1845         st = kmalloc(sizeof(*st), GFP_KERNEL);
1846         if (st == NULL)
1847                 return -ENOMEM;
1848
1849         page_count = obj->base.size / PAGE_SIZE;
1850         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1851                 kfree(st);
1852                 return -ENOMEM;
1853         }
1854
1855         /* Get the list of pages out of our struct file.  They'll be pinned
1856          * at this point until we release them.
1857          *
1858          * Fail silently without starting the shrinker
1859          */
1860         mapping = file_inode(obj->base.filp)->i_mapping;
1861         gfp = mapping_gfp_mask(mapping);
1862         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1863         gfp &= ~(__GFP_IO | __GFP_WAIT);
1864         sg = st->sgl;
1865         st->nents = 0;
1866         for (i = 0; i < page_count; i++) {
1867                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1868                 if (IS_ERR(page)) {
1869                         i915_gem_purge(dev_priv, page_count);
1870                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1871                 }
1872                 if (IS_ERR(page)) {
1873                         /* We've tried hard to allocate the memory by reaping
1874                          * our own buffer, now let the real VM do its job and
1875                          * go down in flames if truly OOM.
1876                          */
1877                         gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
1878                         gfp |= __GFP_IO | __GFP_WAIT;
1879
1880                         i915_gem_shrink_all(dev_priv);
1881                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1882                         if (IS_ERR(page))
1883                                 goto err_pages;
1884
1885                         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1886                         gfp &= ~(__GFP_IO | __GFP_WAIT);
1887                 }
1888 #ifdef CONFIG_SWIOTLB
1889                 if (swiotlb_nr_tbl()) {
1890                         st->nents++;
1891                         sg_set_page(sg, page, PAGE_SIZE, 0);
1892                         sg = sg_next(sg);
1893                         continue;
1894                 }
1895 #endif
1896                 if (!i || page_to_pfn(page) != last_pfn + 1) {
1897                         if (i)
1898                                 sg = sg_next(sg);
1899                         st->nents++;
1900                         sg_set_page(sg, page, PAGE_SIZE, 0);
1901                 } else {
1902                         sg->length += PAGE_SIZE;
1903                 }
1904                 last_pfn = page_to_pfn(page);
1905
1906                 /* Check that the i965g/gm workaround works. */
1907                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
1908         }
1909 #ifdef CONFIG_SWIOTLB
1910         if (!swiotlb_nr_tbl())
1911 #endif
1912                 sg_mark_end(sg);
1913         obj->pages = st;
1914
1915         if (i915_gem_object_needs_bit17_swizzle(obj))
1916                 i915_gem_object_do_bit_17_swizzle(obj);
1917
1918         return 0;
1919
1920 err_pages:
1921         sg_mark_end(sg);
1922         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1923                 page_cache_release(sg_page_iter_page(&sg_iter));
1924         sg_free_table(st);
1925         kfree(st);
1926         return PTR_ERR(page);
1927 }
1928
1929 /* Ensure that the associated pages are gathered from the backing storage
1930  * and pinned into our object. i915_gem_object_get_pages() may be called
1931  * multiple times before they are released by a single call to
1932  * i915_gem_object_put_pages() - once the pages are no longer referenced
1933  * either as a result of memory pressure (reaping pages under the shrinker)
1934  * or as the object is itself released.
1935  */
1936 int
1937 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1938 {
1939         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1940         const struct drm_i915_gem_object_ops *ops = obj->ops;
1941         int ret;
1942
1943         if (obj->pages)
1944                 return 0;
1945
1946         if (obj->madv != I915_MADV_WILLNEED) {
1947                 DRM_ERROR("Attempting to obtain a purgeable object\n");
1948                 return -EINVAL;
1949         }
1950
1951         BUG_ON(obj->pages_pin_count);
1952
1953         ret = ops->get_pages(obj);
1954         if (ret)
1955                 return ret;
1956
1957         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1958         return 0;
1959 }
1960
1961 static void
1962 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1963                                struct intel_ring_buffer *ring)
1964 {
1965         struct drm_device *dev = obj->base.dev;
1966         struct drm_i915_private *dev_priv = dev->dev_private;
1967         u32 seqno = intel_ring_get_seqno(ring);
1968
1969         BUG_ON(ring == NULL);
1970         if (obj->ring != ring && obj->last_write_seqno) {
1971                 /* Keep the seqno relative to the current ring */
1972                 obj->last_write_seqno = seqno;
1973         }
1974         obj->ring = ring;
1975
1976         /* Add a reference if we're newly entering the active list. */
1977         if (!obj->active) {
1978                 drm_gem_object_reference(&obj->base);
1979                 obj->active = 1;
1980         }
1981
1982         list_move_tail(&obj->ring_list, &ring->active_list);
1983
1984         obj->last_read_seqno = seqno;
1985
1986         if (obj->fenced_gpu_access) {
1987                 obj->last_fenced_seqno = seqno;
1988
1989                 /* Bump MRU to take account of the delayed flush */
1990                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1991                         struct drm_i915_fence_reg *reg;
1992
1993                         reg = &dev_priv->fence_regs[obj->fence_reg];
1994                         list_move_tail(&reg->lru_list,
1995                                        &dev_priv->mm.fence_list);
1996                 }
1997         }
1998 }
1999
2000 void i915_vma_move_to_active(struct i915_vma *vma,
2001                              struct intel_ring_buffer *ring)
2002 {
2003         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2004         return i915_gem_object_move_to_active(vma->obj, ring);
2005 }
2006
2007 static void
2008 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2009 {
2010         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2011         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
2012         struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2013
2014         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2015         BUG_ON(!obj->active);
2016
2017         list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2018
2019         list_del_init(&obj->ring_list);
2020         obj->ring = NULL;
2021
2022         obj->last_read_seqno = 0;
2023         obj->last_write_seqno = 0;
2024         obj->base.write_domain = 0;
2025
2026         obj->last_fenced_seqno = 0;
2027         obj->fenced_gpu_access = false;
2028
2029         obj->active = 0;
2030         drm_gem_object_unreference(&obj->base);
2031
2032         WARN_ON(i915_verify_lists(dev));
2033 }
2034
2035 static int
2036 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2037 {
2038         struct drm_i915_private *dev_priv = dev->dev_private;
2039         struct intel_ring_buffer *ring;
2040         int ret, i, j;
2041
2042         /* Carefully retire all requests without writing to the rings */
2043         for_each_ring(ring, dev_priv, i) {
2044                 ret = intel_ring_idle(ring);
2045                 if (ret)
2046                         return ret;
2047         }
2048         i915_gem_retire_requests(dev);
2049
2050         /* Finally reset hw state */
2051         for_each_ring(ring, dev_priv, i) {
2052                 intel_ring_init_seqno(ring, seqno);
2053
2054                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
2055                         ring->sync_seqno[j] = 0;
2056         }
2057
2058         return 0;
2059 }
2060
2061 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2062 {
2063         struct drm_i915_private *dev_priv = dev->dev_private;
2064         int ret;
2065
2066         if (seqno == 0)
2067                 return -EINVAL;
2068
2069         /* HWS page needs to be set less than what we
2070          * will inject to ring
2071          */
2072         ret = i915_gem_init_seqno(dev, seqno - 1);
2073         if (ret)
2074                 return ret;
2075
2076         /* Carefully set the last_seqno value so that wrap
2077          * detection still works
2078          */
2079         dev_priv->next_seqno = seqno;
2080         dev_priv->last_seqno = seqno - 1;
2081         if (dev_priv->last_seqno == 0)
2082                 dev_priv->last_seqno--;
2083
2084         return 0;
2085 }
2086
2087 int
2088 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2089 {
2090         struct drm_i915_private *dev_priv = dev->dev_private;
2091
2092         /* reserve 0 for non-seqno */
2093         if (dev_priv->next_seqno == 0) {
2094                 int ret = i915_gem_init_seqno(dev, 0);
2095                 if (ret)
2096                         return ret;
2097
2098                 dev_priv->next_seqno = 1;
2099         }
2100
2101         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2102         return 0;
2103 }
2104
2105 int __i915_add_request(struct intel_ring_buffer *ring,
2106                        struct drm_file *file,
2107                        struct drm_i915_gem_object *obj,
2108                        u32 *out_seqno)
2109 {
2110         drm_i915_private_t *dev_priv = ring->dev->dev_private;
2111         struct drm_i915_gem_request *request;
2112         u32 request_ring_position, request_start;
2113         int was_empty;
2114         int ret;
2115
2116         request_start = intel_ring_get_tail(ring);
2117         /*
2118          * Emit any outstanding flushes - execbuf can fail to emit the flush
2119          * after having emitted the batchbuffer command. Hence we need to fix
2120          * things up similar to emitting the lazy request. The difference here
2121          * is that the flush _must_ happen before the next request, no matter
2122          * what.
2123          */
2124         ret = intel_ring_flush_all_caches(ring);
2125         if (ret)
2126                 return ret;
2127
2128         request = ring->preallocated_lazy_request;
2129         if (WARN_ON(request == NULL))
2130                 return -ENOMEM;
2131
2132         /* Record the position of the start of the request so that
2133          * should we detect the updated seqno part-way through the
2134          * GPU processing the request, we never over-estimate the
2135          * position of the head.
2136          */
2137         request_ring_position = intel_ring_get_tail(ring);
2138
2139         ret = ring->add_request(ring);
2140         if (ret)
2141                 return ret;
2142
2143         request->seqno = intel_ring_get_seqno(ring);
2144         request->ring = ring;
2145         request->head = request_start;
2146         request->tail = request_ring_position;
2147
2148         /* Whilst this request exists, batch_obj will be on the
2149          * active_list, and so will hold the active reference. Only when this
2150          * request is retired will the the batch_obj be moved onto the
2151          * inactive_list and lose its active reference. Hence we do not need
2152          * to explicitly hold another reference here.
2153          */
2154         request->batch_obj = obj;
2155
2156         /* Hold a reference to the current context so that we can inspect
2157          * it later in case a hangcheck error event fires.
2158          */
2159         request->ctx = ring->last_context;
2160         if (request->ctx)
2161                 i915_gem_context_reference(request->ctx);
2162
2163         request->emitted_jiffies = jiffies;
2164         was_empty = list_empty(&ring->request_list);
2165         list_add_tail(&request->list, &ring->request_list);
2166         request->file_priv = NULL;
2167
2168         if (file) {
2169                 struct drm_i915_file_private *file_priv = file->driver_priv;
2170
2171                 spin_lock(&file_priv->mm.lock);
2172                 request->file_priv = file_priv;
2173                 list_add_tail(&request->client_list,
2174                               &file_priv->mm.request_list);
2175                 spin_unlock(&file_priv->mm.lock);
2176         }
2177
2178         trace_i915_gem_request_add(ring, request->seqno);
2179         ring->outstanding_lazy_seqno = 0;
2180         ring->preallocated_lazy_request = NULL;
2181
2182         if (!dev_priv->ums.mm_suspended) {
2183                 i915_queue_hangcheck(ring->dev);
2184
2185                 if (was_empty) {
2186                         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2187                         queue_delayed_work(dev_priv->wq,
2188                                            &dev_priv->mm.retire_work,
2189                                            round_jiffies_up_relative(HZ));
2190                         intel_mark_busy(dev_priv->dev);
2191                 }
2192         }
2193
2194         if (out_seqno)
2195                 *out_seqno = request->seqno;
2196         return 0;
2197 }
2198
2199 static inline void
2200 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2201 {
2202         struct drm_i915_file_private *file_priv = request->file_priv;
2203
2204         if (!file_priv)
2205                 return;
2206
2207         spin_lock(&file_priv->mm.lock);
2208         list_del(&request->client_list);
2209         request->file_priv = NULL;
2210         spin_unlock(&file_priv->mm.lock);
2211 }
2212
2213 static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
2214                                     struct i915_address_space *vm)
2215 {
2216         if (acthd >= i915_gem_obj_offset(obj, vm) &&
2217             acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
2218                 return true;
2219
2220         return false;
2221 }
2222
2223 static bool i915_head_inside_request(const u32 acthd_unmasked,
2224                                      const u32 request_start,
2225                                      const u32 request_end)
2226 {
2227         const u32 acthd = acthd_unmasked & HEAD_ADDR;
2228
2229         if (request_start < request_end) {
2230                 if (acthd >= request_start && acthd < request_end)
2231                         return true;
2232         } else if (request_start > request_end) {
2233                 if (acthd >= request_start || acthd < request_end)
2234                         return true;
2235         }
2236
2237         return false;
2238 }
2239
2240 static struct i915_address_space *
2241 request_to_vm(struct drm_i915_gem_request *request)
2242 {
2243         struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
2244         struct i915_address_space *vm;
2245
2246         vm = &dev_priv->gtt.base;
2247
2248         return vm;
2249 }
2250
2251 static bool i915_request_guilty(struct drm_i915_gem_request *request,
2252                                 const u32 acthd, bool *inside)
2253 {
2254         /* There is a possibility that unmasked head address
2255          * pointing inside the ring, matches the batch_obj address range.
2256          * However this is extremely unlikely.
2257          */
2258         if (request->batch_obj) {
2259                 if (i915_head_inside_object(acthd, request->batch_obj,
2260                                             request_to_vm(request))) {
2261                         *inside = true;
2262                         return true;
2263                 }
2264         }
2265
2266         if (i915_head_inside_request(acthd, request->head, request->tail)) {
2267                 *inside = false;
2268                 return true;
2269         }
2270
2271         return false;
2272 }
2273
2274 static bool i915_context_is_banned(const struct i915_ctx_hang_stats *hs)
2275 {
2276         const unsigned long elapsed = get_seconds() - hs->guilty_ts;
2277
2278         if (hs->banned)
2279                 return true;
2280
2281         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2282                 DRM_ERROR("context hanging too fast, declaring banned!\n");
2283                 return true;
2284         }
2285
2286         return false;
2287 }
2288
2289 static void i915_set_reset_status(struct intel_ring_buffer *ring,
2290                                   struct drm_i915_gem_request *request,
2291                                   u32 acthd)
2292 {
2293         struct i915_ctx_hang_stats *hs = NULL;
2294         bool inside, guilty;
2295         unsigned long offset = 0;
2296
2297         /* Innocent until proven guilty */
2298         guilty = false;
2299
2300         if (request->batch_obj)
2301                 offset = i915_gem_obj_offset(request->batch_obj,
2302                                              request_to_vm(request));
2303
2304         if (ring->hangcheck.action != HANGCHECK_WAIT &&
2305             i915_request_guilty(request, acthd, &inside)) {
2306                 DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
2307                           ring->name,
2308                           inside ? "inside" : "flushing",
2309                           offset,
2310                           request->ctx ? request->ctx->id : 0,
2311                           acthd);
2312
2313                 guilty = true;
2314         }
2315
2316         /* If contexts are disabled or this is the default context, use
2317          * file_priv->reset_state
2318          */
2319         if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
2320                 hs = &request->ctx->hang_stats;
2321         else if (request->file_priv)
2322                 hs = &request->file_priv->hang_stats;
2323
2324         if (hs) {
2325                 if (guilty) {
2326                         hs->banned = i915_context_is_banned(hs);
2327                         hs->batch_active++;
2328                         hs->guilty_ts = get_seconds();
2329                 } else {
2330                         hs->batch_pending++;
2331                 }
2332         }
2333 }
2334
2335 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2336 {
2337         list_del(&request->list);
2338         i915_gem_request_remove_from_client(request);
2339
2340         if (request->ctx)
2341                 i915_gem_context_unreference(request->ctx);
2342
2343         kfree(request);
2344 }
2345
2346 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2347                                        struct intel_ring_buffer *ring)
2348 {
2349         u32 completed_seqno = ring->get_seqno(ring, false);
2350         u32 acthd = intel_ring_get_active_head(ring);
2351         struct drm_i915_gem_request *request;
2352
2353         list_for_each_entry(request, &ring->request_list, list) {
2354                 if (i915_seqno_passed(completed_seqno, request->seqno))
2355                         continue;
2356
2357                 i915_set_reset_status(ring, request, acthd);
2358         }
2359 }
2360
2361 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2362                                         struct intel_ring_buffer *ring)
2363 {
2364         while (!list_empty(&ring->request_list)) {
2365                 struct drm_i915_gem_request *request;
2366
2367                 request = list_first_entry(&ring->request_list,
2368                                            struct drm_i915_gem_request,
2369                                            list);
2370
2371                 i915_gem_free_request(request);
2372         }
2373
2374         while (!list_empty(&ring->active_list)) {
2375                 struct drm_i915_gem_object *obj;
2376
2377                 obj = list_first_entry(&ring->active_list,
2378                                        struct drm_i915_gem_object,
2379                                        ring_list);
2380
2381                 i915_gem_object_move_to_inactive(obj);
2382         }
2383 }
2384
2385 void i915_gem_restore_fences(struct drm_device *dev)
2386 {
2387         struct drm_i915_private *dev_priv = dev->dev_private;
2388         int i;
2389
2390         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2391                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2392
2393                 /*
2394                  * Commit delayed tiling changes if we have an object still
2395                  * attached to the fence, otherwise just clear the fence.
2396                  */
2397                 if (reg->obj) {
2398                         i915_gem_object_update_fence(reg->obj, reg,
2399                                                      reg->obj->tiling_mode);
2400                 } else {
2401                         i915_gem_write_fence(dev, i, NULL);
2402                 }
2403         }
2404 }
2405
2406 void i915_gem_reset(struct drm_device *dev)
2407 {
2408         struct drm_i915_private *dev_priv = dev->dev_private;
2409         struct intel_ring_buffer *ring;
2410         int i;
2411
2412         /*
2413          * Before we free the objects from the requests, we need to inspect
2414          * them for finding the guilty party. As the requests only borrow
2415          * their reference to the objects, the inspection must be done first.
2416          */
2417         for_each_ring(ring, dev_priv, i)
2418                 i915_gem_reset_ring_status(dev_priv, ring);
2419
2420         for_each_ring(ring, dev_priv, i)
2421                 i915_gem_reset_ring_cleanup(dev_priv, ring);
2422
2423         i915_gem_cleanup_ringbuffer(dev);
2424
2425         i915_gem_restore_fences(dev);
2426 }
2427
2428 /**
2429  * This function clears the request list as sequence numbers are passed.
2430  */
2431 void
2432 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2433 {
2434         uint32_t seqno;
2435
2436         if (list_empty(&ring->request_list))
2437                 return;
2438
2439         WARN_ON(i915_verify_lists(ring->dev));
2440
2441         seqno = ring->get_seqno(ring, true);
2442
2443         while (!list_empty(&ring->request_list)) {
2444                 struct drm_i915_gem_request *request;
2445
2446                 request = list_first_entry(&ring->request_list,
2447                                            struct drm_i915_gem_request,
2448                                            list);
2449
2450                 if (!i915_seqno_passed(seqno, request->seqno))
2451                         break;
2452
2453                 trace_i915_gem_request_retire(ring, request->seqno);
2454                 /* We know the GPU must have read the request to have
2455                  * sent us the seqno + interrupt, so use the position
2456                  * of tail of the request to update the last known position
2457                  * of the GPU head.
2458                  */
2459                 ring->last_retired_head = request->tail;
2460
2461                 i915_gem_free_request(request);
2462         }
2463
2464         /* Move any buffers on the active list that are no longer referenced
2465          * by the ringbuffer to the flushing/inactive lists as appropriate.
2466          */
2467         while (!list_empty(&ring->active_list)) {
2468                 struct drm_i915_gem_object *obj;
2469
2470                 obj = list_first_entry(&ring->active_list,
2471                                       struct drm_i915_gem_object,
2472                                       ring_list);
2473
2474                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2475                         break;
2476
2477                 i915_gem_object_move_to_inactive(obj);
2478         }
2479
2480         if (unlikely(ring->trace_irq_seqno &&
2481                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2482                 ring->irq_put(ring);
2483                 ring->trace_irq_seqno = 0;
2484         }
2485
2486         WARN_ON(i915_verify_lists(ring->dev));
2487 }
2488
2489 bool
2490 i915_gem_retire_requests(struct drm_device *dev)
2491 {
2492         drm_i915_private_t *dev_priv = dev->dev_private;
2493         struct intel_ring_buffer *ring;
2494         bool idle = true;
2495         int i;
2496
2497         for_each_ring(ring, dev_priv, i) {
2498                 i915_gem_retire_requests_ring(ring);
2499                 idle &= list_empty(&ring->request_list);
2500         }
2501
2502         if (idle)
2503                 mod_delayed_work(dev_priv->wq,
2504                                    &dev_priv->mm.idle_work,
2505                                    msecs_to_jiffies(100));
2506
2507         return idle;
2508 }
2509
2510 static void
2511 i915_gem_retire_work_handler(struct work_struct *work)
2512 {
2513         struct drm_i915_private *dev_priv =
2514                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2515         struct drm_device *dev = dev_priv->dev;
2516         bool idle;
2517
2518         /* Come back later if the device is busy... */
2519         idle = false;
2520         if (mutex_trylock(&dev->struct_mutex)) {
2521                 idle = i915_gem_retire_requests(dev);
2522                 mutex_unlock(&dev->struct_mutex);
2523         }
2524         if (!idle)
2525                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2526                                    round_jiffies_up_relative(HZ));
2527 }
2528
2529 static void
2530 i915_gem_idle_work_handler(struct work_struct *work)
2531 {
2532         struct drm_i915_private *dev_priv =
2533                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2534
2535         intel_mark_idle(dev_priv->dev);
2536 }
2537
2538 /**
2539  * Ensures that an object will eventually get non-busy by flushing any required
2540  * write domains, emitting any outstanding lazy request and retiring and
2541  * completed requests.
2542  */
2543 static int
2544 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2545 {
2546         int ret;
2547
2548         if (obj->active) {
2549                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2550                 if (ret)
2551                         return ret;
2552
2553                 i915_gem_retire_requests_ring(obj->ring);
2554         }
2555
2556         return 0;
2557 }
2558
2559 /**
2560  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2561  * @DRM_IOCTL_ARGS: standard ioctl arguments
2562  *
2563  * Returns 0 if successful, else an error is returned with the remaining time in
2564  * the timeout parameter.
2565  *  -ETIME: object is still busy after timeout
2566  *  -ERESTARTSYS: signal interrupted the wait
2567  *  -ENONENT: object doesn't exist
2568  * Also possible, but rare:
2569  *  -EAGAIN: GPU wedged
2570  *  -ENOMEM: damn
2571  *  -ENODEV: Internal IRQ fail
2572  *  -E?: The add request failed
2573  *
2574  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2575  * non-zero timeout parameter the wait ioctl will wait for the given number of
2576  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2577  * without holding struct_mutex the object may become re-busied before this
2578  * function completes. A similar but shorter * race condition exists in the busy
2579  * ioctl
2580  */
2581 int
2582 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2583 {
2584         drm_i915_private_t *dev_priv = dev->dev_private;
2585         struct drm_i915_gem_wait *args = data;
2586         struct drm_i915_gem_object *obj;
2587         struct intel_ring_buffer *ring = NULL;
2588         struct timespec timeout_stack, *timeout = NULL;
2589         unsigned reset_counter;
2590         u32 seqno = 0;
2591         int ret = 0;
2592
2593         if (args->timeout_ns >= 0) {
2594                 timeout_stack = ns_to_timespec(args->timeout_ns);
2595                 timeout = &timeout_stack;
2596         }
2597
2598         ret = i915_mutex_lock_interruptible(dev);
2599         if (ret)
2600                 return ret;
2601
2602         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2603         if (&obj->base == NULL) {
2604                 mutex_unlock(&dev->struct_mutex);
2605                 return -ENOENT;
2606         }
2607
2608         /* Need to make sure the object gets inactive eventually. */
2609         ret = i915_gem_object_flush_active(obj);
2610         if (ret)
2611                 goto out;
2612
2613         if (obj->active) {
2614                 seqno = obj->last_read_seqno;
2615                 ring = obj->ring;
2616         }
2617
2618         if (seqno == 0)
2619                  goto out;
2620
2621         /* Do this after OLR check to make sure we make forward progress polling
2622          * on this IOCTL with a 0 timeout (like busy ioctl)
2623          */
2624         if (!args->timeout_ns) {
2625                 ret = -ETIME;
2626                 goto out;
2627         }
2628
2629         drm_gem_object_unreference(&obj->base);
2630         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2631         mutex_unlock(&dev->struct_mutex);
2632
2633         ret = __wait_seqno(ring, seqno, reset_counter, true, timeout, file->driver_priv);
2634         if (timeout)
2635                 args->timeout_ns = timespec_to_ns(timeout);
2636         return ret;
2637
2638 out:
2639         drm_gem_object_unreference(&obj->base);
2640         mutex_unlock(&dev->struct_mutex);
2641         return ret;
2642 }
2643
2644 /**
2645  * i915_gem_object_sync - sync an object to a ring.
2646  *
2647  * @obj: object which may be in use on another ring.
2648  * @to: ring we wish to use the object on. May be NULL.
2649  *
2650  * This code is meant to abstract object synchronization with the GPU.
2651  * Calling with NULL implies synchronizing the object with the CPU
2652  * rather than a particular GPU ring.
2653  *
2654  * Returns 0 if successful, else propagates up the lower layer error.
2655  */
2656 int
2657 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2658                      struct intel_ring_buffer *to)
2659 {
2660         struct intel_ring_buffer *from = obj->ring;
2661         u32 seqno;
2662         int ret, idx;
2663
2664         if (from == NULL || to == from)
2665                 return 0;
2666
2667         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2668                 return i915_gem_object_wait_rendering(obj, false);
2669
2670         idx = intel_ring_sync_index(from, to);
2671
2672         seqno = obj->last_read_seqno;
2673         if (seqno <= from->sync_seqno[idx])
2674                 return 0;
2675
2676         ret = i915_gem_check_olr(obj->ring, seqno);
2677         if (ret)
2678                 return ret;
2679
2680         trace_i915_gem_ring_sync_to(from, to, seqno);
2681         ret = to->sync_to(to, from, seqno);
2682         if (!ret)
2683                 /* We use last_read_seqno because sync_to()
2684                  * might have just caused seqno wrap under
2685                  * the radar.
2686                  */
2687                 from->sync_seqno[idx] = obj->last_read_seqno;
2688
2689         return ret;
2690 }
2691
2692 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2693 {
2694         u32 old_write_domain, old_read_domains;
2695
2696         /* Force a pagefault for domain tracking on next user access */
2697         i915_gem_release_mmap(obj);
2698
2699         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2700                 return;
2701
2702         /* Wait for any direct GTT access to complete */
2703         mb();
2704
2705         old_read_domains = obj->base.read_domains;
2706         old_write_domain = obj->base.write_domain;
2707
2708         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2709         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2710
2711         trace_i915_gem_object_change_domain(obj,
2712                                             old_read_domains,
2713                                             old_write_domain);
2714 }
2715
2716 int i915_vma_unbind(struct i915_vma *vma)
2717 {
2718         struct drm_i915_gem_object *obj = vma->obj;
2719         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2720         int ret;
2721
2722         /* For now we only ever use 1 vma per object */
2723         WARN_ON(!list_is_singular(&obj->vma_list));
2724
2725         if (list_empty(&vma->vma_link))
2726                 return 0;
2727
2728         if (!drm_mm_node_allocated(&vma->node)) {
2729                 i915_gem_vma_destroy(vma);
2730
2731                 return 0;
2732         }
2733
2734         if (obj->pin_count)
2735                 return -EBUSY;
2736
2737         BUG_ON(obj->pages == NULL);
2738
2739         ret = i915_gem_object_finish_gpu(obj);
2740         if (ret)
2741                 return ret;
2742         /* Continue on if we fail due to EIO, the GPU is hung so we
2743          * should be safe and we need to cleanup or else we might
2744          * cause memory corruption through use-after-free.
2745          */
2746
2747         i915_gem_object_finish_gtt(obj);
2748
2749         /* release the fence reg _after_ flushing */
2750         ret = i915_gem_object_put_fence(obj);
2751         if (ret)
2752                 return ret;
2753
2754         trace_i915_vma_unbind(vma);
2755
2756         if (obj->has_global_gtt_mapping)
2757                 i915_gem_gtt_unbind_object(obj);
2758         if (obj->has_aliasing_ppgtt_mapping) {
2759                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2760                 obj->has_aliasing_ppgtt_mapping = 0;
2761         }
2762         i915_gem_gtt_finish_object(obj);
2763         i915_gem_object_unpin_pages(obj);
2764
2765         list_del(&vma->mm_list);
2766         /* Avoid an unnecessary call to unbind on rebind. */
2767         if (i915_is_ggtt(vma->vm))
2768                 obj->map_and_fenceable = true;
2769
2770         drm_mm_remove_node(&vma->node);
2771
2772         i915_gem_vma_destroy(vma);
2773
2774         /* Since the unbound list is global, only move to that list if
2775          * no more VMAs exist. */
2776         if (list_empty(&obj->vma_list))
2777                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2778
2779         return 0;
2780 }
2781
2782 /**
2783  * Unbinds an object from the global GTT aperture.
2784  */
2785 int
2786 i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
2787 {
2788         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2789         struct i915_address_space *ggtt = &dev_priv->gtt.base;
2790
2791         if (!i915_gem_obj_ggtt_bound(obj))
2792                 return 0;
2793
2794         if (obj->pin_count)
2795                 return -EBUSY;
2796
2797         BUG_ON(obj->pages == NULL);
2798
2799         return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
2800 }
2801
2802 int i915_gpu_idle(struct drm_device *dev)
2803 {
2804         drm_i915_private_t *dev_priv = dev->dev_private;
2805         struct intel_ring_buffer *ring;
2806         int ret, i;
2807
2808         /* Flush everything onto the inactive list. */
2809         for_each_ring(ring, dev_priv, i) {
2810                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2811                 if (ret)
2812                         return ret;
2813
2814                 ret = intel_ring_idle(ring);
2815                 if (ret)
2816                         return ret;
2817         }
2818
2819         return 0;
2820 }
2821
2822 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2823                                  struct drm_i915_gem_object *obj)
2824 {
2825         drm_i915_private_t *dev_priv = dev->dev_private;
2826         int fence_reg;
2827         int fence_pitch_shift;
2828
2829         if (INTEL_INFO(dev)->gen >= 6) {
2830                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
2831                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
2832         } else {
2833                 fence_reg = FENCE_REG_965_0;
2834                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
2835         }
2836
2837         fence_reg += reg * 8;
2838
2839         /* To w/a incoherency with non-atomic 64-bit register updates,
2840          * we split the 64-bit update into two 32-bit writes. In order
2841          * for a partial fence not to be evaluated between writes, we
2842          * precede the update with write to turn off the fence register,
2843          * and only enable the fence as the last step.
2844          *
2845          * For extra levels of paranoia, we make sure each step lands
2846          * before applying the next step.
2847          */
2848         I915_WRITE(fence_reg, 0);
2849         POSTING_READ(fence_reg);
2850
2851         if (obj) {
2852                 u32 size = i915_gem_obj_ggtt_size(obj);
2853                 uint64_t val;
2854
2855                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2856                                  0xfffff000) << 32;
2857                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2858                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2859                 if (obj->tiling_mode == I915_TILING_Y)
2860                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2861                 val |= I965_FENCE_REG_VALID;
2862
2863                 I915_WRITE(fence_reg + 4, val >> 32);
2864                 POSTING_READ(fence_reg + 4);
2865
2866                 I915_WRITE(fence_reg + 0, val);
2867                 POSTING_READ(fence_reg);
2868         } else {
2869                 I915_WRITE(fence_reg + 4, 0);
2870                 POSTING_READ(fence_reg + 4);
2871         }
2872 }
2873
2874 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2875                                  struct drm_i915_gem_object *obj)
2876 {
2877         drm_i915_private_t *dev_priv = dev->dev_private;
2878         u32 val;
2879
2880         if (obj) {
2881                 u32 size = i915_gem_obj_ggtt_size(obj);
2882                 int pitch_val;
2883                 int tile_width;
2884
2885                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2886                      (size & -size) != size ||
2887                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2888                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2889                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2890
2891                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2892                         tile_width = 128;
2893                 else
2894                         tile_width = 512;
2895
2896                 /* Note: pitch better be a power of two tile widths */
2897                 pitch_val = obj->stride / tile_width;
2898                 pitch_val = ffs(pitch_val) - 1;
2899
2900                 val = i915_gem_obj_ggtt_offset(obj);
2901                 if (obj->tiling_mode == I915_TILING_Y)
2902                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2903                 val |= I915_FENCE_SIZE_BITS(size);
2904                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2905                 val |= I830_FENCE_REG_VALID;
2906         } else
2907                 val = 0;
2908
2909         if (reg < 8)
2910                 reg = FENCE_REG_830_0 + reg * 4;
2911         else
2912                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2913
2914         I915_WRITE(reg, val);
2915         POSTING_READ(reg);
2916 }
2917
2918 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2919                                 struct drm_i915_gem_object *obj)
2920 {
2921         drm_i915_private_t *dev_priv = dev->dev_private;
2922         uint32_t val;
2923
2924         if (obj) {
2925                 u32 size = i915_gem_obj_ggtt_size(obj);
2926                 uint32_t pitch_val;
2927
2928                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2929                      (size & -size) != size ||
2930                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2931                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
2932                      i915_gem_obj_ggtt_offset(obj), size);
2933
2934                 pitch_val = obj->stride / 128;
2935                 pitch_val = ffs(pitch_val) - 1;
2936
2937                 val = i915_gem_obj_ggtt_offset(obj);
2938                 if (obj->tiling_mode == I915_TILING_Y)
2939                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2940                 val |= I830_FENCE_SIZE_BITS(size);
2941                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2942                 val |= I830_FENCE_REG_VALID;
2943         } else
2944                 val = 0;
2945
2946         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2947         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2948 }
2949
2950 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
2951 {
2952         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
2953 }
2954
2955 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2956                                  struct drm_i915_gem_object *obj)
2957 {
2958         struct drm_i915_private *dev_priv = dev->dev_private;
2959
2960         /* Ensure that all CPU reads are completed before installing a fence
2961          * and all writes before removing the fence.
2962          */
2963         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
2964                 mb();
2965
2966         WARN(obj && (!obj->stride || !obj->tiling_mode),
2967              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
2968              obj->stride, obj->tiling_mode);
2969
2970         switch (INTEL_INFO(dev)->gen) {
2971         case 8:
2972         case 7:
2973         case 6:
2974         case 5:
2975         case 4: i965_write_fence_reg(dev, reg, obj); break;
2976         case 3: i915_write_fence_reg(dev, reg, obj); break;
2977         case 2: i830_write_fence_reg(dev, reg, obj); break;
2978         default: BUG();
2979         }
2980
2981         /* And similarly be paranoid that no direct access to this region
2982          * is reordered to before the fence is installed.
2983          */
2984         if (i915_gem_object_needs_mb(obj))
2985                 mb();
2986 }
2987
2988 static inline int fence_number(struct drm_i915_private *dev_priv,
2989                                struct drm_i915_fence_reg *fence)
2990 {
2991         return fence - dev_priv->fence_regs;
2992 }
2993
2994 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2995                                          struct drm_i915_fence_reg *fence,
2996                                          bool enable)
2997 {
2998         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2999         int reg = fence_number(dev_priv, fence);
3000
3001         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3002
3003         if (enable) {
3004                 obj->fence_reg = reg;
3005                 fence->obj = obj;
3006                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
3007         } else {
3008                 obj->fence_reg = I915_FENCE_REG_NONE;
3009                 fence->obj = NULL;
3010                 list_del_init(&fence->lru_list);
3011         }
3012         obj->fence_dirty = false;
3013 }
3014
3015 static int
3016 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3017 {
3018         if (obj->last_fenced_seqno) {
3019                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
3020                 if (ret)
3021                         return ret;
3022
3023                 obj->last_fenced_seqno = 0;
3024         }
3025
3026         obj->fenced_gpu_access = false;
3027         return 0;
3028 }
3029
3030 int
3031 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3032 {
3033         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3034         struct drm_i915_fence_reg *fence;
3035         int ret;
3036
3037         ret = i915_gem_object_wait_fence(obj);
3038         if (ret)
3039                 return ret;
3040
3041         if (obj->fence_reg == I915_FENCE_REG_NONE)
3042                 return 0;
3043
3044         fence = &dev_priv->fence_regs[obj->fence_reg];
3045
3046         i915_gem_object_fence_lost(obj);
3047         i915_gem_object_update_fence(obj, fence, false);
3048
3049         return 0;
3050 }
3051
3052 static struct drm_i915_fence_reg *
3053 i915_find_fence_reg(struct drm_device *dev)
3054 {
3055         struct drm_i915_private *dev_priv = dev->dev_private;
3056         struct drm_i915_fence_reg *reg, *avail;
3057         int i;
3058
3059         /* First try to find a free reg */
3060         avail = NULL;
3061         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3062                 reg = &dev_priv->fence_regs[i];
3063                 if (!reg->obj)
3064                         return reg;
3065
3066                 if (!reg->pin_count)
3067                         avail = reg;
3068         }
3069
3070         if (avail == NULL)
3071                 return NULL;
3072
3073         /* None available, try to steal one or wait for a user to finish */
3074         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3075                 if (reg->pin_count)
3076                         continue;
3077
3078                 return reg;
3079         }
3080
3081         return NULL;
3082 }
3083
3084 /**
3085  * i915_gem_object_get_fence - set up fencing for an object
3086  * @obj: object to map through a fence reg
3087  *
3088  * When mapping objects through the GTT, userspace wants to be able to write
3089  * to them without having to worry about swizzling if the object is tiled.
3090  * This function walks the fence regs looking for a free one for @obj,
3091  * stealing one if it can't find any.
3092  *
3093  * It then sets up the reg based on the object's properties: address, pitch
3094  * and tiling format.
3095  *
3096  * For an untiled surface, this removes any existing fence.
3097  */
3098 int
3099 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3100 {
3101         struct drm_device *dev = obj->base.dev;
3102         struct drm_i915_private *dev_priv = dev->dev_private;
3103         bool enable = obj->tiling_mode != I915_TILING_NONE;
3104         struct drm_i915_fence_reg *reg;
3105         int ret;
3106
3107         /* Have we updated the tiling parameters upon the object and so
3108          * will need to serialise the write to the associated fence register?
3109          */
3110         if (obj->fence_dirty) {
3111                 ret = i915_gem_object_wait_fence(obj);
3112                 if (ret)
3113                         return ret;
3114         }
3115
3116         /* Just update our place in the LRU if our fence is getting reused. */
3117         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3118                 reg = &dev_priv->fence_regs[obj->fence_reg];
3119                 if (!obj->fence_dirty) {
3120                         list_move_tail(&reg->lru_list,
3121                                        &dev_priv->mm.fence_list);
3122                         return 0;
3123                 }
3124         } else if (enable) {
3125                 reg = i915_find_fence_reg(dev);
3126                 if (reg == NULL)
3127                         return -EDEADLK;
3128
3129                 if (reg->obj) {
3130                         struct drm_i915_gem_object *old = reg->obj;
3131
3132                         ret = i915_gem_object_wait_fence(old);
3133                         if (ret)
3134                                 return ret;
3135
3136                         i915_gem_object_fence_lost(old);
3137                 }
3138         } else
3139                 return 0;
3140
3141         i915_gem_object_update_fence(obj, reg, enable);
3142
3143         return 0;
3144 }
3145
3146 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
3147                                      struct drm_mm_node *gtt_space,
3148                                      unsigned long cache_level)
3149 {
3150         struct drm_mm_node *other;
3151
3152         /* On non-LLC machines we have to be careful when putting differing
3153          * types of snoopable memory together to avoid the prefetcher
3154          * crossing memory domains and dying.
3155          */
3156         if (HAS_LLC(dev))
3157                 return true;
3158
3159         if (!drm_mm_node_allocated(gtt_space))
3160                 return true;
3161
3162         if (list_empty(&gtt_space->node_list))
3163                 return true;
3164
3165         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3166         if (other->allocated && !other->hole_follows && other->color != cache_level)
3167                 return false;
3168
3169         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3170         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3171                 return false;
3172
3173         return true;
3174 }
3175
3176 static void i915_gem_verify_gtt(struct drm_device *dev)
3177 {
3178 #if WATCH_GTT
3179         struct drm_i915_private *dev_priv = dev->dev_private;
3180         struct drm_i915_gem_object *obj;
3181         int err = 0;
3182
3183         list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3184                 if (obj->gtt_space == NULL) {
3185                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
3186                         err++;
3187                         continue;
3188                 }
3189
3190                 if (obj->cache_level != obj->gtt_space->color) {
3191                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3192                                i915_gem_obj_ggtt_offset(obj),
3193                                i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3194                                obj->cache_level,
3195                                obj->gtt_space->color);
3196                         err++;
3197                         continue;
3198                 }
3199
3200                 if (!i915_gem_valid_gtt_space(dev,
3201                                               obj->gtt_space,
3202                                               obj->cache_level)) {
3203                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3204                                i915_gem_obj_ggtt_offset(obj),
3205                                i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3206                                obj->cache_level);
3207                         err++;
3208                         continue;
3209                 }
3210         }
3211
3212         WARN_ON(err);
3213 #endif
3214 }
3215
3216 /**
3217  * Finds free space in the GTT aperture and binds the object there.
3218  */
3219 static int
3220 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3221                            struct i915_address_space *vm,
3222                            unsigned alignment,
3223                            bool map_and_fenceable,
3224                            bool nonblocking)
3225 {
3226         struct drm_device *dev = obj->base.dev;
3227         drm_i915_private_t *dev_priv = dev->dev_private;
3228         u32 size, fence_size, fence_alignment, unfenced_alignment;
3229         size_t gtt_max =
3230                 map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
3231         struct i915_vma *vma;
3232         int ret;
3233
3234         fence_size = i915_gem_get_gtt_size(dev,
3235                                            obj->base.size,
3236                                            obj->tiling_mode);
3237         fence_alignment = i915_gem_get_gtt_alignment(dev,
3238                                                      obj->base.size,
3239                                                      obj->tiling_mode, true);
3240         unfenced_alignment =
3241                 i915_gem_get_gtt_alignment(dev,
3242                                                     obj->base.size,
3243                                                     obj->tiling_mode, false);
3244
3245         if (alignment == 0)
3246                 alignment = map_and_fenceable ? fence_alignment :
3247                                                 unfenced_alignment;
3248         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3249                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
3250                 return -EINVAL;
3251         }
3252
3253         size = map_and_fenceable ? fence_size : obj->base.size;
3254
3255         /* If the object is bigger than the entire aperture, reject it early
3256          * before evicting everything in a vain attempt to find space.
3257          */
3258         if (obj->base.size > gtt_max) {
3259                 DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3260                           obj->base.size,
3261                           map_and_fenceable ? "mappable" : "total",
3262                           gtt_max);
3263                 return -E2BIG;
3264         }
3265
3266         ret = i915_gem_object_get_pages(obj);
3267         if (ret)
3268                 return ret;
3269
3270         i915_gem_object_pin_pages(obj);
3271
3272         BUG_ON(!i915_is_ggtt(vm));
3273
3274         vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
3275         if (IS_ERR(vma)) {
3276                 ret = PTR_ERR(vma);
3277                 goto err_unpin;
3278         }
3279
3280         /* For now we only ever use 1 vma per object */
3281         WARN_ON(!list_is_singular(&obj->vma_list));
3282
3283 search_free:
3284         ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3285                                                   size, alignment,
3286                                                   obj->cache_level, 0, gtt_max,
3287                                                   DRM_MM_SEARCH_DEFAULT);
3288         if (ret) {
3289                 ret = i915_gem_evict_something(dev, vm, size, alignment,
3290                                                obj->cache_level,
3291                                                map_and_fenceable,
3292                                                nonblocking);
3293                 if (ret == 0)
3294                         goto search_free;
3295
3296                 goto err_free_vma;
3297         }
3298         if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
3299                                               obj->cache_level))) {
3300                 ret = -EINVAL;
3301                 goto err_remove_node;
3302         }
3303
3304         ret = i915_gem_gtt_prepare_object(obj);
3305         if (ret)
3306                 goto err_remove_node;
3307
3308         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3309         list_add_tail(&vma->mm_list, &vm->inactive_list);
3310
3311         if (i915_is_ggtt(vm)) {
3312                 bool mappable, fenceable;
3313
3314                 fenceable = (vma->node.size == fence_size &&
3315                              (vma->node.start & (fence_alignment - 1)) == 0);
3316
3317                 mappable = (vma->node.start + obj->base.size <=
3318                             dev_priv->gtt.mappable_end);
3319
3320                 obj->map_and_fenceable = mappable && fenceable;
3321         }
3322
3323         WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
3324
3325         trace_i915_vma_bind(vma, map_and_fenceable);
3326         i915_gem_verify_gtt(dev);
3327         return 0;
3328
3329 err_remove_node:
3330         drm_mm_remove_node(&vma->node);
3331 err_free_vma:
3332         i915_gem_vma_destroy(vma);
3333 err_unpin:
3334         i915_gem_object_unpin_pages(obj);
3335         return ret;
3336 }
3337
3338 bool
3339 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3340                         bool force)
3341 {
3342         /* If we don't have a page list set up, then we're not pinned
3343          * to GPU, and we can ignore the cache flush because it'll happen
3344          * again at bind time.
3345          */
3346         if (obj->pages == NULL)
3347                 return false;
3348
3349         /*
3350          * Stolen memory is always coherent with the GPU as it is explicitly
3351          * marked as wc by the system, or the system is cache-coherent.
3352          */
3353         if (obj->stolen)
3354                 return false;
3355
3356         /* If the GPU is snooping the contents of the CPU cache,
3357          * we do not need to manually clear the CPU cache lines.  However,
3358          * the caches are only snooped when the render cache is
3359          * flushed/invalidated.  As we always have to emit invalidations
3360          * and flushes when moving into and out of the RENDER domain, correct
3361          * snooping behaviour occurs naturally as the result of our domain
3362          * tracking.
3363          */
3364         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3365                 return false;
3366
3367         trace_i915_gem_object_clflush(obj);
3368         drm_clflush_sg(obj->pages);
3369
3370         return true;
3371 }
3372
3373 /** Flushes the GTT write domain for the object if it's dirty. */
3374 static void
3375 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3376 {
3377         uint32_t old_write_domain;
3378
3379         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3380                 return;
3381
3382         /* No actual flushing is required for the GTT write domain.  Writes
3383          * to it immediately go to main memory as far as we know, so there's
3384          * no chipset flush.  It also doesn't land in render cache.
3385          *
3386          * However, we do have to enforce the order so that all writes through
3387          * the GTT land before any writes to the device, such as updates to
3388          * the GATT itself.
3389          */
3390         wmb();
3391
3392         old_write_domain = obj->base.write_domain;
3393         obj->base.write_domain = 0;
3394
3395         trace_i915_gem_object_change_domain(obj,
3396                                             obj->base.read_domains,
3397                                             old_write_domain);
3398 }
3399
3400 /** Flushes the CPU write domain for the object if it's dirty. */
3401 static void
3402 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
3403                                        bool force)
3404 {
3405         uint32_t old_write_domain;
3406
3407         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3408                 return;
3409
3410         if (i915_gem_clflush_object(obj, force))
3411                 i915_gem_chipset_flush(obj->base.dev);
3412
3413         old_write_domain = obj->base.write_domain;
3414         obj->base.write_domain = 0;
3415
3416         trace_i915_gem_object_change_domain(obj,
3417                                             obj->base.read_domains,
3418                                             old_write_domain);
3419 }
3420
3421 /**
3422  * Moves a single object to the GTT read, and possibly write domain.
3423  *
3424  * This function returns when the move is complete, including waiting on
3425  * flushes to occur.
3426  */
3427 int
3428 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3429 {
3430         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3431         uint32_t old_write_domain, old_read_domains;
3432         int ret;
3433
3434         /* Not valid to be called on unbound objects. */
3435         if (!i915_gem_obj_bound_any(obj))
3436                 return -EINVAL;
3437
3438         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3439                 return 0;
3440
3441         ret = i915_gem_object_wait_rendering(obj, !write);
3442         if (ret)
3443                 return ret;
3444
3445         i915_gem_object_flush_cpu_write_domain(obj, false);
3446
3447         /* Serialise direct access to this object with the barriers for
3448          * coherent writes from the GPU, by effectively invalidating the
3449          * GTT domain upon first access.
3450          */
3451         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3452                 mb();
3453
3454         old_write_domain = obj->base.write_domain;
3455         old_read_domains = obj->base.read_domains;
3456
3457         /* It should now be out of any other write domains, and we can update
3458          * the domain values for our changes.
3459          */
3460         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3461         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3462         if (write) {
3463                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3464                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3465                 obj->dirty = 1;
3466         }
3467
3468         trace_i915_gem_object_change_domain(obj,
3469                                             old_read_domains,
3470                                             old_write_domain);
3471
3472         /* And bump the LRU for this access */
3473         if (i915_gem_object_is_inactive(obj)) {
3474                 struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
3475                 if (vma)
3476                         list_move_tail(&vma->mm_list,
3477                                        &dev_priv->gtt.base.inactive_list);
3478
3479         }
3480
3481         return 0;
3482 }
3483
3484 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3485                                     enum i915_cache_level cache_level)
3486 {
3487         struct drm_device *dev = obj->base.dev;
3488         drm_i915_private_t *dev_priv = dev->dev_private;
3489         struct i915_vma *vma;
3490         int ret;
3491
3492         if (obj->cache_level == cache_level)
3493                 return 0;
3494
3495         if (obj->pin_count) {
3496                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3497                 return -EBUSY;
3498         }
3499
3500         list_for_each_entry(vma, &obj->vma_list, vma_link) {
3501                 if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
3502                         ret = i915_vma_unbind(vma);
3503                         if (ret)
3504                                 return ret;
3505
3506                         break;
3507                 }
3508         }
3509
3510         if (i915_gem_obj_bound_any(obj)) {
3511                 ret = i915_gem_object_finish_gpu(obj);
3512                 if (ret)
3513                         return ret;
3514
3515                 i915_gem_object_finish_gtt(obj);
3516
3517                 /* Before SandyBridge, you could not use tiling or fence
3518                  * registers with snooped memory, so relinquish any fences
3519                  * currently pointing to our region in the aperture.
3520                  */
3521                 if (INTEL_INFO(dev)->gen < 6) {
3522                         ret = i915_gem_object_put_fence(obj);
3523                         if (ret)
3524                                 return ret;
3525                 }
3526
3527                 if (obj->has_global_gtt_mapping)
3528                         i915_gem_gtt_bind_object(obj, cache_level);
3529                 if (obj->has_aliasing_ppgtt_mapping)
3530                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3531                                                obj, cache_level);
3532         }
3533
3534         list_for_each_entry(vma, &obj->vma_list, vma_link)
3535                 vma->node.color = cache_level;
3536         obj->cache_level = cache_level;
3537
3538         if (cpu_write_needs_clflush(obj)) {
3539                 u32 old_read_domains, old_write_domain;
3540
3541                 /* If we're coming from LLC cached, then we haven't
3542                  * actually been tracking whether the data is in the
3543                  * CPU cache or not, since we only allow one bit set
3544                  * in obj->write_domain and have been skipping the clflushes.
3545                  * Just set it to the CPU cache for now.
3546                  */
3547                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3548
3549                 old_read_domains = obj->base.read_domains;
3550                 old_write_domain = obj->base.write_domain;
3551
3552                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3553                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3554
3555                 trace_i915_gem_object_change_domain(obj,
3556                                                     old_read_domains,
3557                                                     old_write_domain);
3558         }
3559
3560         i915_gem_verify_gtt(dev);
3561         return 0;
3562 }
3563
3564 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3565                                struct drm_file *file)
3566 {
3567         struct drm_i915_gem_caching *args = data;
3568         struct drm_i915_gem_object *obj;
3569         int ret;
3570
3571         ret = i915_mutex_lock_interruptible(dev);
3572         if (ret)
3573                 return ret;
3574
3575         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3576         if (&obj->base == NULL) {
3577                 ret = -ENOENT;
3578                 goto unlock;
3579         }
3580
3581         switch (obj->cache_level) {
3582         case I915_CACHE_LLC:
3583         case I915_CACHE_L3_LLC:
3584                 args->caching = I915_CACHING_CACHED;
3585                 break;
3586
3587         case I915_CACHE_WT:
3588                 args->caching = I915_CACHING_DISPLAY;
3589                 break;
3590
3591         default:
3592                 args->caching = I915_CACHING_NONE;
3593                 break;
3594         }
3595
3596         drm_gem_object_unreference(&obj->base);
3597 unlock:
3598         mutex_unlock(&dev->struct_mutex);
3599         return ret;
3600 }
3601
3602 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3603                                struct drm_file *file)
3604 {
3605         struct drm_i915_gem_caching *args = data;
3606         struct drm_i915_gem_object *obj;
3607         enum i915_cache_level level;
3608         int ret;
3609
3610         switch (args->caching) {
3611         case I915_CACHING_NONE:
3612                 level = I915_CACHE_NONE;
3613                 break;
3614         case I915_CACHING_CACHED:
3615                 level = I915_CACHE_LLC;
3616                 break;
3617         case I915_CACHING_DISPLAY:
3618                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3619                 break;
3620         default:
3621                 return -EINVAL;
3622         }
3623
3624         ret = i915_mutex_lock_interruptible(dev);
3625         if (ret)
3626                 return ret;
3627
3628         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3629         if (&obj->base == NULL) {
3630                 ret = -ENOENT;
3631                 goto unlock;
3632         }
3633
3634         ret = i915_gem_object_set_cache_level(obj, level);
3635
3636         drm_gem_object_unreference(&obj->base);
3637 unlock:
3638         mutex_unlock(&dev->struct_mutex);
3639         return ret;
3640 }
3641
3642 static bool is_pin_display(struct drm_i915_gem_object *obj)
3643 {
3644         /* There are 3 sources that pin objects:
3645          *   1. The display engine (scanouts, sprites, cursors);
3646          *   2. Reservations for execbuffer;
3647          *   3. The user.
3648          *
3649          * We can ignore reservations as we hold the struct_mutex and
3650          * are only called outside of the reservation path.  The user
3651          * can only increment pin_count once, and so if after
3652          * subtracting the potential reference by the user, any pin_count
3653          * remains, it must be due to another use by the display engine.
3654          */
3655         return obj->pin_count - !!obj->user_pin_count;
3656 }
3657
3658 /*
3659  * Prepare buffer for display plane (scanout, cursors, etc).
3660  * Can be called from an uninterruptible phase (modesetting) and allows
3661  * any flushes to be pipelined (for pageflips).
3662  */
3663 int
3664 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3665                                      u32 alignment,
3666                                      struct intel_ring_buffer *pipelined)
3667 {
3668         u32 old_read_domains, old_write_domain;
3669         int ret;
3670
3671         if (pipelined != obj->ring) {
3672                 ret = i915_gem_object_sync(obj, pipelined);
3673                 if (ret)
3674                         return ret;
3675         }
3676
3677         /* Mark the pin_display early so that we account for the
3678          * display coherency whilst setting up the cache domains.
3679          */
3680         obj->pin_display = true;
3681
3682         /* The display engine is not coherent with the LLC cache on gen6.  As
3683          * a result, we make sure that the pinning that is about to occur is
3684          * done with uncached PTEs. This is lowest common denominator for all
3685          * chipsets.
3686          *
3687          * However for gen6+, we could do better by using the GFDT bit instead
3688          * of uncaching, which would allow us to flush all the LLC-cached data
3689          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3690          */
3691         ret = i915_gem_object_set_cache_level(obj,
3692                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3693         if (ret)
3694                 goto err_unpin_display;
3695
3696         /* As the user may map the buffer once pinned in the display plane
3697          * (e.g. libkms for the bootup splash), we have to ensure that we
3698          * always use map_and_fenceable for all scanout buffers.
3699          */
3700         ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
3701         if (ret)
3702                 goto err_unpin_display;
3703
3704         i915_gem_object_flush_cpu_write_domain(obj, true);
3705
3706         old_write_domain = obj->base.write_domain;
3707         old_read_domains = obj->base.read_domains;
3708
3709         /* It should now be out of any other write domains, and we can update
3710          * the domain values for our changes.
3711          */
3712         obj->base.write_domain = 0;
3713         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3714
3715         trace_i915_gem_object_change_domain(obj,
3716                                             old_read_domains,
3717                                             old_write_domain);
3718
3719         return 0;
3720
3721 err_unpin_display:
3722         obj->pin_display = is_pin_display(obj);
3723         return ret;
3724 }
3725
3726 void
3727 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
3728 {
3729         i915_gem_object_unpin(obj);
3730         obj->pin_display = is_pin_display(obj);
3731 }
3732
3733 int
3734 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3735 {
3736         int ret;
3737
3738         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3739                 return 0;
3740
3741         ret = i915_gem_object_wait_rendering(obj, false);
3742         if (ret)
3743                 return ret;
3744
3745         /* Ensure that we invalidate the GPU's caches and TLBs. */
3746         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3747         return 0;
3748 }
3749
3750 /**
3751  * Moves a single object to the CPU read, and possibly write domain.
3752  *
3753  * This function returns when the move is complete, including waiting on
3754  * flushes to occur.
3755  */
3756 int
3757 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3758 {
3759         uint32_t old_write_domain, old_read_domains;
3760         int ret;
3761
3762         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3763                 return 0;
3764
3765         ret = i915_gem_object_wait_rendering(obj, !write);
3766         if (ret)
3767                 return ret;
3768
3769         i915_gem_object_flush_gtt_write_domain(obj);
3770
3771         old_write_domain = obj->base.write_domain;
3772         old_read_domains = obj->base.read_domains;
3773
3774         /* Flush the CPU cache if it's still invalid. */
3775         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3776                 i915_gem_clflush_object(obj, false);
3777
3778                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3779         }
3780
3781         /* It should now be out of any other write domains, and we can update
3782          * the domain values for our changes.
3783          */
3784         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3785
3786         /* If we're writing through the CPU, then the GPU read domains will
3787          * need to be invalidated at next use.
3788          */
3789         if (write) {
3790                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3791                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3792         }
3793
3794         trace_i915_gem_object_change_domain(obj,
3795                                             old_read_domains,
3796                                             old_write_domain);
3797
3798         return 0;
3799 }
3800
3801 /* Throttle our rendering by waiting until the ring has completed our requests
3802  * emitted over 20 msec ago.
3803  *
3804  * Note that if we were to use the current jiffies each time around the loop,
3805  * we wouldn't escape the function with any frames outstanding if the time to
3806  * render a frame was over 20ms.
3807  *
3808  * This should get us reasonable parallelism between CPU and GPU but also
3809  * relatively low latency when blocking on a particular request to finish.
3810  */
3811 static int
3812 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3813 {
3814         struct drm_i915_private *dev_priv = dev->dev_private;
3815         struct drm_i915_file_private *file_priv = file->driver_priv;
3816         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3817         struct drm_i915_gem_request *request;
3818         struct intel_ring_buffer *ring = NULL;
3819         unsigned reset_counter;
3820         u32 seqno = 0;
3821         int ret;
3822
3823         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3824         if (ret)
3825                 return ret;
3826
3827         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
3828         if (ret)
3829                 return ret;
3830
3831         spin_lock(&file_priv->mm.lock);
3832         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3833                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3834                         break;
3835
3836                 ring = request->ring;
3837                 seqno = request->seqno;
3838         }
3839         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3840         spin_unlock(&file_priv->mm.lock);
3841
3842         if (seqno == 0)
3843                 return 0;
3844
3845         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
3846         if (ret == 0)
3847                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3848
3849         return ret;
3850 }
3851
3852 int
3853 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3854                     struct i915_address_space *vm,
3855                     uint32_t alignment,
3856                     bool map_and_fenceable,
3857                     bool nonblocking)
3858 {
3859         struct i915_vma *vma;
3860         int ret;
3861
3862         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3863                 return -EBUSY;
3864
3865         WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
3866
3867         vma = i915_gem_obj_to_vma(obj, vm);
3868
3869         if (vma) {
3870                 if ((alignment &&
3871                      vma->node.start & (alignment - 1)) ||
3872                     (map_and_fenceable && !obj->map_and_fenceable)) {
3873                         WARN(obj->pin_count,
3874                              "bo is already pinned with incorrect alignment:"
3875                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3876                              " obj->map_and_fenceable=%d\n",
3877                              i915_gem_obj_offset(obj, vm), alignment,
3878                              map_and_fenceable,
3879                              obj->map_and_fenceable);
3880                         ret = i915_vma_unbind(vma);
3881                         if (ret)
3882                                 return ret;
3883                 }
3884         }
3885
3886         if (!i915_gem_obj_bound(obj, vm)) {
3887                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3888
3889                 ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3890                                                  map_and_fenceable,
3891                                                  nonblocking);
3892                 if (ret)
3893                         return ret;
3894
3895                 if (!dev_priv->mm.aliasing_ppgtt)
3896                         i915_gem_gtt_bind_object(obj, obj->cache_level);
3897         }
3898
3899         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3900                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3901
3902         obj->pin_count++;
3903         obj->pin_mappable |= map_and_fenceable;
3904
3905         return 0;
3906 }
3907
3908 void
3909 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3910 {
3911         BUG_ON(obj->pin_count == 0);
3912         BUG_ON(!i915_gem_obj_bound_any(obj));
3913
3914         if (--obj->pin_count == 0)
3915                 obj->pin_mappable = false;
3916 }
3917
3918 int
3919 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3920                    struct drm_file *file)
3921 {
3922         struct drm_i915_gem_pin *args = data;
3923         struct drm_i915_gem_object *obj;
3924         int ret;
3925
3926         ret = i915_mutex_lock_interruptible(dev);
3927         if (ret)
3928                 return ret;
3929
3930         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3931         if (&obj->base == NULL) {
3932                 ret = -ENOENT;
3933                 goto unlock;
3934         }
3935
3936         if (obj->madv != I915_MADV_WILLNEED) {
3937                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3938                 ret = -EINVAL;
3939                 goto out;
3940         }
3941
3942         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3943                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3944                           args->handle);
3945                 ret = -EINVAL;
3946                 goto out;
3947         }
3948
3949         if (obj->user_pin_count == ULONG_MAX) {
3950                 ret = -EBUSY;
3951                 goto out;
3952         }
3953
3954         if (obj->user_pin_count == 0) {
3955                 ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3956                 if (ret)
3957                         goto out;
3958         }
3959
3960         obj->user_pin_count++;
3961         obj->pin_filp = file;
3962
3963         args->offset = i915_gem_obj_ggtt_offset(obj);
3964 out:
3965         drm_gem_object_unreference(&obj->base);
3966 unlock:
3967         mutex_unlock(&dev->struct_mutex);
3968         return ret;
3969 }
3970
3971 int
3972 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3973                      struct drm_file *file)
3974 {
3975         struct drm_i915_gem_pin *args = data;
3976         struct drm_i915_gem_object *obj;
3977         int ret;
3978
3979         ret = i915_mutex_lock_interruptible(dev);
3980         if (ret)
3981                 return ret;
3982
3983         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3984         if (&obj->base == NULL) {
3985                 ret = -ENOENT;
3986                 goto unlock;
3987         }
3988
3989         if (obj->pin_filp != file) {
3990                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3991                           args->handle);
3992                 ret = -EINVAL;
3993                 goto out;
3994         }
3995         obj->user_pin_count--;
3996         if (obj->user_pin_count == 0) {
3997                 obj->pin_filp = NULL;
3998                 i915_gem_object_unpin(obj);
3999         }
4000
4001 out:
4002         drm_gem_object_unreference(&obj->base);
4003 unlock:
4004         mutex_unlock(&dev->struct_mutex);
4005         return ret;
4006 }
4007
4008 int
4009 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4010                     struct drm_file *file)
4011 {
4012         struct drm_i915_gem_busy *args = data;
4013         struct drm_i915_gem_object *obj;
4014         int ret;
4015
4016         ret = i915_mutex_lock_interruptible(dev);
4017         if (ret)
4018                 return ret;
4019
4020         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4021         if (&obj->base == NULL) {
4022                 ret = -ENOENT;
4023                 goto unlock;
4024         }
4025
4026         /* Count all active objects as busy, even if they are currently not used
4027          * by the gpu. Users of this interface expect objects to eventually
4028          * become non-busy without any further actions, therefore emit any
4029          * necessary flushes here.
4030          */
4031         ret = i915_gem_object_flush_active(obj);
4032
4033         args->busy = obj->active;
4034         if (obj->ring) {
4035                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
4036                 args->busy |= intel_ring_flag(obj->ring) << 16;
4037         }
4038
4039         drm_gem_object_unreference(&obj->base);
4040 unlock:
4041         mutex_unlock(&dev->struct_mutex);
4042         return ret;
4043 }
4044
4045 int
4046 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4047                         struct drm_file *file_priv)
4048 {
4049         return i915_gem_ring_throttle(dev, file_priv);
4050 }
4051
4052 int
4053 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4054                        struct drm_file *file_priv)
4055 {
4056         struct drm_i915_gem_madvise *args = data;
4057         struct drm_i915_gem_object *obj;
4058         int ret;
4059
4060         switch (args->madv) {
4061         case I915_MADV_DONTNEED:
4062         case I915_MADV_WILLNEED:
4063             break;
4064         default:
4065             return -EINVAL;
4066         }
4067
4068         ret = i915_mutex_lock_interruptible(dev);
4069         if (ret)
4070                 return ret;
4071
4072         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4073         if (&obj->base == NULL) {
4074                 ret = -ENOENT;
4075                 goto unlock;
4076         }
4077
4078         if (obj->pin_count) {
4079                 ret = -EINVAL;
4080                 goto out;
4081         }
4082
4083         if (obj->madv != __I915_MADV_PURGED)
4084                 obj->madv = args->madv;
4085
4086         /* if the object is no longer attached, discard its backing storage */
4087         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
4088                 i915_gem_object_truncate(obj);
4089
4090         args->retained = obj->madv != __I915_MADV_PURGED;
4091
4092 out:
4093         drm_gem_object_unreference(&obj->base);
4094 unlock:
4095         mutex_unlock(&dev->struct_mutex);
4096         return ret;
4097 }
4098
4099 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4100                           const struct drm_i915_gem_object_ops *ops)
4101 {
4102         INIT_LIST_HEAD(&obj->global_list);
4103         INIT_LIST_HEAD(&obj->ring_list);
4104         INIT_LIST_HEAD(&obj->obj_exec_link);
4105         INIT_LIST_HEAD(&obj->vma_list);
4106
4107         obj->ops = ops;
4108
4109         obj->fence_reg = I915_FENCE_REG_NONE;
4110         obj->madv = I915_MADV_WILLNEED;
4111         /* Avoid an unnecessary call to unbind on the first bind. */
4112         obj->map_and_fenceable = true;
4113
4114         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4115 }
4116
4117 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4118         .get_pages = i915_gem_object_get_pages_gtt,
4119         .put_pages = i915_gem_object_put_pages_gtt,
4120 };
4121
4122 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4123                                                   size_t size)
4124 {
4125         struct drm_i915_gem_object *obj;
4126         struct address_space *mapping;
4127         gfp_t mask;
4128
4129         obj = i915_gem_object_alloc(dev);
4130         if (obj == NULL)
4131                 return NULL;
4132
4133         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4134                 i915_gem_object_free(obj);
4135                 return NULL;
4136         }
4137
4138         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4139         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4140                 /* 965gm cannot relocate objects above 4GiB. */
4141                 mask &= ~__GFP_HIGHMEM;
4142                 mask |= __GFP_DMA32;
4143         }
4144
4145         mapping = file_inode(obj->base.filp)->i_mapping;
4146         mapping_set_gfp_mask(mapping, mask);
4147
4148         i915_gem_object_init(obj, &i915_gem_object_ops);
4149
4150         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4151         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4152
4153         if (HAS_LLC(dev)) {
4154                 /* On some devices, we can have the GPU use the LLC (the CPU
4155                  * cache) for about a 10% performance improvement
4156                  * compared to uncached.  Graphics requests other than
4157                  * display scanout are coherent with the CPU in
4158                  * accessing this cache.  This means in this mode we
4159                  * don't need to clflush on the CPU side, and on the
4160                  * GPU side we only need to flush internal caches to
4161                  * get data visible to the CPU.
4162                  *
4163                  * However, we maintain the display planes as UC, and so
4164                  * need to rebind when first used as such.
4165                  */
4166                 obj->cache_level = I915_CACHE_LLC;
4167         } else
4168                 obj->cache_level = I915_CACHE_NONE;
4169
4170         trace_i915_gem_object_create(obj);
4171
4172         return obj;
4173 }
4174
4175 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4176 {
4177         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4178         struct drm_device *dev = obj->base.dev;
4179         drm_i915_private_t *dev_priv = dev->dev_private;
4180         struct i915_vma *vma, *next;
4181
4182         trace_i915_gem_object_destroy(obj);
4183
4184         if (obj->phys_obj)
4185                 i915_gem_detach_phys_object(dev, obj);
4186
4187         obj->pin_count = 0;
4188         /* NB: 0 or 1 elements */
4189         WARN_ON(!list_empty(&obj->vma_list) &&
4190                 !list_is_singular(&obj->vma_list));
4191         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4192                 int ret = i915_vma_unbind(vma);
4193                 if (WARN_ON(ret == -ERESTARTSYS)) {
4194                         bool was_interruptible;
4195
4196                         was_interruptible = dev_priv->mm.interruptible;
4197                         dev_priv->mm.interruptible = false;
4198
4199                         WARN_ON(i915_vma_unbind(vma));
4200
4201                         dev_priv->mm.interruptible = was_interruptible;
4202                 }
4203         }
4204
4205         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4206          * before progressing. */
4207         if (obj->stolen)
4208                 i915_gem_object_unpin_pages(obj);
4209
4210         if (WARN_ON(obj->pages_pin_count))
4211                 obj->pages_pin_count = 0;
4212         i915_gem_object_put_pages(obj);
4213         i915_gem_object_free_mmap_offset(obj);
4214         i915_gem_object_release_stolen(obj);
4215
4216         BUG_ON(obj->pages);
4217
4218         if (obj->base.import_attach)
4219                 drm_prime_gem_destroy(&obj->base, NULL);
4220
4221         drm_gem_object_release(&obj->base);
4222         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4223
4224         kfree(obj->bit_17);
4225         i915_gem_object_free(obj);
4226 }
4227
4228 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4229                                      struct i915_address_space *vm)
4230 {
4231         struct i915_vma *vma;
4232         list_for_each_entry(vma, &obj->vma_list, vma_link)
4233                 if (vma->vm == vm)
4234                         return vma;
4235
4236         return NULL;
4237 }
4238
4239 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
4240                                               struct i915_address_space *vm)
4241 {
4242         struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
4243         if (vma == NULL)
4244                 return ERR_PTR(-ENOMEM);
4245
4246         INIT_LIST_HEAD(&vma->vma_link);
4247         INIT_LIST_HEAD(&vma->mm_list);
4248         INIT_LIST_HEAD(&vma->exec_list);
4249         vma->vm = vm;
4250         vma->obj = obj;
4251
4252         /* Keep GGTT vmas first to make debug easier */
4253         if (i915_is_ggtt(vm))
4254                 list_add(&vma->vma_link, &obj->vma_list);
4255         else
4256                 list_add_tail(&vma->vma_link, &obj->vma_list);
4257
4258         return vma;
4259 }
4260
4261 struct i915_vma *
4262 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
4263                                   struct i915_address_space *vm)
4264 {
4265         struct i915_vma *vma;
4266
4267         vma = i915_gem_obj_to_vma(obj, vm);
4268         if (!vma)
4269                 vma = __i915_gem_vma_create(obj, vm);
4270
4271         return vma;
4272 }
4273
4274 void i915_gem_vma_destroy(struct i915_vma *vma)
4275 {
4276         WARN_ON(vma->node.allocated);
4277
4278         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4279         if (!list_empty(&vma->exec_list))
4280                 return;
4281
4282         list_del(&vma->vma_link);
4283
4284         kfree(vma);
4285 }
4286
4287 int
4288 i915_gem_suspend(struct drm_device *dev)
4289 {
4290         drm_i915_private_t *dev_priv = dev->dev_private;
4291         int ret = 0;
4292
4293         mutex_lock(&dev->struct_mutex);
4294         if (dev_priv->ums.mm_suspended)
4295                 goto err;
4296
4297         ret = i915_gpu_idle(dev);
4298         if (ret)
4299                 goto err;
4300
4301         i915_gem_retire_requests(dev);
4302
4303         /* Under UMS, be paranoid and evict. */
4304         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4305                 i915_gem_evict_everything(dev);
4306
4307         i915_kernel_lost_context(dev);
4308         i915_gem_cleanup_ringbuffer(dev);
4309
4310         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4311          * We need to replace this with a semaphore, or something.
4312          * And not confound ums.mm_suspended!
4313          */
4314         dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
4315                                                              DRIVER_MODESET);
4316         mutex_unlock(&dev->struct_mutex);
4317
4318         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4319         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4320         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
4321
4322         return 0;
4323
4324 err:
4325         mutex_unlock(&dev->struct_mutex);
4326         return ret;
4327 }
4328
4329 int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice)
4330 {
4331         struct drm_device *dev = ring->dev;
4332         drm_i915_private_t *dev_priv = dev->dev_private;
4333         u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
4334         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4335         int i, ret;
4336
4337         if (!HAS_L3_DPF(dev) || !remap_info)
4338                 return 0;
4339
4340         ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
4341         if (ret)
4342                 return ret;
4343
4344         /*
4345          * Note: We do not worry about the concurrent register cacheline hang
4346          * here because no other code should access these registers other than
4347          * at initialization time.
4348          */
4349         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4350                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4351                 intel_ring_emit(ring, reg_base + i);
4352                 intel_ring_emit(ring, remap_info[i/4]);
4353         }
4354
4355         intel_ring_advance(ring);
4356
4357         return ret;
4358 }
4359
4360 void i915_gem_init_swizzling(struct drm_device *dev)
4361 {
4362         drm_i915_private_t *dev_priv = dev->dev_private;
4363
4364         if (INTEL_INFO(dev)->gen < 5 ||
4365             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4366                 return;
4367
4368         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4369                                  DISP_TILE_SURFACE_SWIZZLING);
4370
4371         if (IS_GEN5(dev))
4372                 return;
4373
4374         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4375         if (IS_GEN6(dev))
4376                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4377         else if (IS_GEN7(dev))
4378                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4379         else if (IS_GEN8(dev))
4380                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4381         else
4382                 BUG();
4383 }
4384
4385 static bool
4386 intel_enable_blt(struct drm_device *dev)
4387 {
4388         if (!HAS_BLT(dev))
4389                 return false;
4390
4391         /* The blitter was dysfunctional on early prototypes */
4392         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4393                 DRM_INFO("BLT not supported on this pre-production hardware;"
4394                          " graphics performance will be degraded.\n");
4395                 return false;
4396         }
4397
4398         return true;
4399 }
4400
4401 static int i915_gem_init_rings(struct drm_device *dev)
4402 {
4403         struct drm_i915_private *dev_priv = dev->dev_private;
4404         int ret;
4405
4406         ret = intel_init_render_ring_buffer(dev);
4407         if (ret)
4408                 return ret;
4409
4410         if (HAS_BSD(dev)) {
4411                 ret = intel_init_bsd_ring_buffer(dev);
4412                 if (ret)
4413                         goto cleanup_render_ring;
4414         }
4415
4416         if (intel_enable_blt(dev)) {
4417                 ret = intel_init_blt_ring_buffer(dev);
4418                 if (ret)
4419                         goto cleanup_bsd_ring;
4420         }
4421
4422         if (HAS_VEBOX(dev)) {
4423                 ret = intel_init_vebox_ring_buffer(dev);
4424                 if (ret)
4425                         goto cleanup_blt_ring;
4426         }
4427
4428
4429         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4430         if (ret)
4431                 goto cleanup_vebox_ring;
4432
4433         return 0;
4434
4435 cleanup_vebox_ring:
4436         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4437 cleanup_blt_ring:
4438         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4439 cleanup_bsd_ring:
4440         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4441 cleanup_render_ring:
4442         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4443
4444         return ret;
4445 }
4446
4447 int
4448 i915_gem_init_hw(struct drm_device *dev)
4449 {
4450         drm_i915_private_t *dev_priv = dev->dev_private;
4451         int ret, i;
4452
4453         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4454                 return -EIO;
4455
4456         if (dev_priv->ellc_size)
4457                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4458
4459         if (IS_HASWELL(dev))
4460                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4461                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4462
4463         if (HAS_PCH_NOP(dev)) {
4464                 u32 temp = I915_READ(GEN7_MSG_CTL);
4465                 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4466                 I915_WRITE(GEN7_MSG_CTL, temp);
4467         }
4468
4469         i915_gem_init_swizzling(dev);
4470
4471         ret = i915_gem_init_rings(dev);
4472         if (ret)
4473                 return ret;
4474
4475         for (i = 0; i < NUM_L3_SLICES(dev); i++)
4476                 i915_gem_l3_remap(&dev_priv->ring[RCS], i);
4477
4478         /*
4479          * XXX: There was some w/a described somewhere suggesting loading
4480          * contexts before PPGTT.
4481          */
4482         i915_gem_context_init(dev);
4483         if (dev_priv->mm.aliasing_ppgtt) {
4484                 ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
4485                 if (ret) {
4486                         i915_gem_cleanup_aliasing_ppgtt(dev);
4487                         DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
4488                 }
4489         }
4490
4491         return 0;
4492 }
4493
4494 int i915_gem_init(struct drm_device *dev)
4495 {
4496         struct drm_i915_private *dev_priv = dev->dev_private;
4497         int ret;
4498
4499         mutex_lock(&dev->struct_mutex);
4500
4501         if (IS_VALLEYVIEW(dev)) {
4502                 /* VLVA0 (potential hack), BIOS isn't actually waking us */
4503                 I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
4504                 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
4505                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4506         }
4507
4508         i915_gem_init_global_gtt(dev);
4509
4510         ret = i915_gem_init_hw(dev);
4511         mutex_unlock(&dev->struct_mutex);
4512         if (ret) {
4513                 i915_gem_cleanup_aliasing_ppgtt(dev);
4514                 return ret;
4515         }
4516
4517         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4518         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4519                 dev_priv->dri1.allow_batchbuffer = 1;
4520         return 0;
4521 }
4522
4523 void
4524 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4525 {
4526         drm_i915_private_t *dev_priv = dev->dev_private;
4527         struct intel_ring_buffer *ring;
4528         int i;
4529
4530         for_each_ring(ring, dev_priv, i)
4531                 intel_cleanup_ring_buffer(ring);
4532 }
4533
4534 int
4535 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4536                        struct drm_file *file_priv)
4537 {
4538         struct drm_i915_private *dev_priv = dev->dev_private;
4539         int ret;
4540
4541         if (drm_core_check_feature(dev, DRIVER_MODESET))
4542                 return 0;
4543
4544         if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4545                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4546                 atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4547         }
4548
4549         mutex_lock(&dev->struct_mutex);
4550         dev_priv->ums.mm_suspended = 0;
4551
4552         ret = i915_gem_init_hw(dev);
4553         if (ret != 0) {
4554                 mutex_unlock(&dev->struct_mutex);
4555                 return ret;
4556         }
4557
4558         BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
4559         mutex_unlock(&dev->struct_mutex);
4560
4561         ret = drm_irq_install(dev);
4562         if (ret)
4563                 goto cleanup_ringbuffer;
4564
4565         return 0;
4566
4567 cleanup_ringbuffer:
4568         mutex_lock(&dev->struct_mutex);
4569         i915_gem_cleanup_ringbuffer(dev);
4570         dev_priv->ums.mm_suspended = 1;
4571         mutex_unlock(&dev->struct_mutex);
4572
4573         return ret;
4574 }
4575
4576 int
4577 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4578                        struct drm_file *file_priv)
4579 {
4580         if (drm_core_check_feature(dev, DRIVER_MODESET))
4581                 return 0;
4582
4583         drm_irq_uninstall(dev);
4584
4585         return i915_gem_suspend(dev);
4586 }
4587
4588 void
4589 i915_gem_lastclose(struct drm_device *dev)
4590 {
4591         int ret;
4592
4593         if (drm_core_check_feature(dev, DRIVER_MODESET))
4594                 return;
4595
4596         ret = i915_gem_suspend(dev);
4597         if (ret)
4598                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4599 }
4600
4601 static void
4602 init_ring_lists(struct intel_ring_buffer *ring)
4603 {
4604         INIT_LIST_HEAD(&ring->active_list);
4605         INIT_LIST_HEAD(&ring->request_list);
4606 }
4607
4608 static void i915_init_vm(struct drm_i915_private *dev_priv,
4609                          struct i915_address_space *vm)
4610 {
4611         vm->dev = dev_priv->dev;
4612         INIT_LIST_HEAD(&vm->active_list);
4613         INIT_LIST_HEAD(&vm->inactive_list);
4614         INIT_LIST_HEAD(&vm->global_link);
4615         list_add(&vm->global_link, &dev_priv->vm_list);
4616 }
4617
4618 void
4619 i915_gem_load(struct drm_device *dev)
4620 {
4621         drm_i915_private_t *dev_priv = dev->dev_private;
4622         int i;
4623
4624         dev_priv->slab =
4625                 kmem_cache_create("i915_gem_object",
4626                                   sizeof(struct drm_i915_gem_object), 0,
4627                                   SLAB_HWCACHE_ALIGN,
4628                                   NULL);
4629
4630         INIT_LIST_HEAD(&dev_priv->vm_list);
4631         i915_init_vm(dev_priv, &dev_priv->gtt.base);
4632
4633         INIT_LIST_HEAD(&dev_priv->context_list);
4634         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4635         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4636         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4637         for (i = 0; i < I915_NUM_RINGS; i++)
4638                 init_ring_lists(&dev_priv->ring[i]);
4639         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4640                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4641         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4642                           i915_gem_retire_work_handler);
4643         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
4644                           i915_gem_idle_work_handler);
4645         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4646
4647         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4648         if (IS_GEN3(dev)) {
4649                 I915_WRITE(MI_ARB_STATE,
4650                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4651         }
4652
4653         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4654
4655         /* Old X drivers will take 0-2 for front, back, depth buffers */
4656         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4657                 dev_priv->fence_reg_start = 3;
4658
4659         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
4660                 dev_priv->num_fence_regs = 32;
4661         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4662                 dev_priv->num_fence_regs = 16;
4663         else
4664                 dev_priv->num_fence_regs = 8;
4665
4666         /* Initialize fence registers to zero */
4667         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4668         i915_gem_restore_fences(dev);
4669
4670         i915_gem_detect_bit_6_swizzle(dev);
4671         init_waitqueue_head(&dev_priv->pending_flip_queue);
4672
4673         dev_priv->mm.interruptible = true;
4674
4675         dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan;
4676         dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count;
4677         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4678         register_shrinker(&dev_priv->mm.inactive_shrinker);
4679 }
4680
4681 /*
4682  * Create a physically contiguous memory object for this object
4683  * e.g. for cursor + overlay regs
4684  */
4685 static int i915_gem_init_phys_object(struct drm_device *dev,
4686                                      int id, int size, int align)
4687 {
4688         drm_i915_private_t *dev_priv = dev->dev_private;
4689         struct drm_i915_gem_phys_object *phys_obj;
4690         int ret;
4691
4692         if (dev_priv->mm.phys_objs[id - 1] || !size)
4693                 return 0;
4694
4695         phys_obj = kzalloc(sizeof(*phys_obj), GFP_KERNEL);
4696         if (!phys_obj)
4697                 return -ENOMEM;
4698
4699         phys_obj->id = id;
4700
4701         phys_obj->handle = drm_pci_alloc(dev, size, align);
4702         if (!phys_obj->handle) {
4703                 ret = -ENOMEM;
4704                 goto kfree_obj;
4705         }
4706 #ifdef CONFIG_X86
4707         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4708 #endif
4709
4710         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4711
4712         return 0;
4713 kfree_obj:
4714         kfree(phys_obj);
4715         return ret;
4716 }
4717
4718 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4719 {
4720         drm_i915_private_t *dev_priv = dev->dev_private;
4721         struct drm_i915_gem_phys_object *phys_obj;
4722
4723         if (!dev_priv->mm.phys_objs[id - 1])
4724                 return;
4725
4726         phys_obj = dev_priv->mm.phys_objs[id - 1];
4727         if (phys_obj->cur_obj) {
4728                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4729         }
4730
4731 #ifdef CONFIG_X86
4732         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4733 #endif
4734         drm_pci_free(dev, phys_obj->handle);
4735         kfree(phys_obj);
4736         dev_priv->mm.phys_objs[id - 1] = NULL;
4737 }
4738
4739 void i915_gem_free_all_phys_object(struct drm_device *dev)
4740 {
4741         int i;
4742
4743         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4744                 i915_gem_free_phys_object(dev, i);
4745 }
4746
4747 void i915_gem_detach_phys_object(struct drm_device *dev,
4748                                  struct drm_i915_gem_object *obj)
4749 {
4750         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4751         char *vaddr;
4752         int i;
4753         int page_count;
4754
4755         if (!obj->phys_obj)
4756                 return;
4757         vaddr = obj->phys_obj->handle->vaddr;
4758
4759         page_count = obj->base.size / PAGE_SIZE;
4760         for (i = 0; i < page_count; i++) {
4761                 struct page *page = shmem_read_mapping_page(mapping, i);
4762                 if (!IS_ERR(page)) {
4763                         char *dst = kmap_atomic(page);
4764                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4765                         kunmap_atomic(dst);
4766
4767                         drm_clflush_pages(&page, 1);
4768
4769                         set_page_dirty(page);
4770                         mark_page_accessed(page);
4771                         page_cache_release(page);
4772                 }
4773         }
4774         i915_gem_chipset_flush(dev);
4775
4776         obj->phys_obj->cur_obj = NULL;
4777         obj->phys_obj = NULL;
4778 }
4779
4780 int
4781 i915_gem_attach_phys_object(struct drm_device *dev,
4782                             struct drm_i915_gem_object *obj,
4783                             int id,
4784                             int align)
4785 {
4786         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4787         drm_i915_private_t *dev_priv = dev->dev_private;
4788         int ret = 0;
4789         int page_count;
4790         int i;
4791
4792         if (id > I915_MAX_PHYS_OBJECT)
4793                 return -EINVAL;
4794
4795         if (obj->phys_obj) {
4796                 if (obj->phys_obj->id == id)
4797                         return 0;
4798                 i915_gem_detach_phys_object(dev, obj);
4799         }
4800
4801         /* create a new object */
4802         if (!dev_priv->mm.phys_objs[id - 1]) {
4803                 ret = i915_gem_init_phys_object(dev, id,
4804                                                 obj->base.size, align);
4805                 if (ret) {
4806                         DRM_ERROR("failed to init phys object %d size: %zu\n",
4807                                   id, obj->base.size);
4808                         return ret;
4809                 }
4810         }
4811
4812         /* bind to the object */
4813         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4814         obj->phys_obj->cur_obj = obj;
4815
4816         page_count = obj->base.size / PAGE_SIZE;
4817
4818         for (i = 0; i < page_count; i++) {
4819                 struct page *page;
4820                 char *dst, *src;
4821
4822                 page = shmem_read_mapping_page(mapping, i);
4823                 if (IS_ERR(page))
4824                         return PTR_ERR(page);
4825
4826                 src = kmap_atomic(page);
4827                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4828                 memcpy(dst, src, PAGE_SIZE);
4829                 kunmap_atomic(src);
4830
4831                 mark_page_accessed(page);
4832                 page_cache_release(page);
4833         }
4834
4835         return 0;
4836 }
4837
4838 static int
4839 i915_gem_phys_pwrite(struct drm_device *dev,
4840                      struct drm_i915_gem_object *obj,
4841                      struct drm_i915_gem_pwrite *args,
4842                      struct drm_file *file_priv)
4843 {
4844         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4845         char __user *user_data = to_user_ptr(args->data_ptr);
4846
4847         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4848                 unsigned long unwritten;
4849
4850                 /* The physical object once assigned is fixed for the lifetime
4851                  * of the obj, so we can safely drop the lock and continue
4852                  * to access vaddr.
4853                  */
4854                 mutex_unlock(&dev->struct_mutex);
4855                 unwritten = copy_from_user(vaddr, user_data, args->size);
4856                 mutex_lock(&dev->struct_mutex);
4857                 if (unwritten)
4858                         return -EFAULT;
4859         }
4860
4861         i915_gem_chipset_flush(dev);
4862         return 0;
4863 }
4864
4865 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4866 {
4867         struct drm_i915_file_private *file_priv = file->driver_priv;
4868
4869         cancel_delayed_work_sync(&file_priv->mm.idle_work);
4870
4871         /* Clean up our request list when the client is going away, so that
4872          * later retire_requests won't dereference our soon-to-be-gone
4873          * file_priv.
4874          */
4875         spin_lock(&file_priv->mm.lock);
4876         while (!list_empty(&file_priv->mm.request_list)) {
4877                 struct drm_i915_gem_request *request;
4878
4879                 request = list_first_entry(&file_priv->mm.request_list,
4880                                            struct drm_i915_gem_request,
4881                                            client_list);
4882                 list_del(&request->client_list);
4883                 request->file_priv = NULL;
4884         }
4885         spin_unlock(&file_priv->mm.lock);
4886 }
4887
4888 static void
4889 i915_gem_file_idle_work_handler(struct work_struct *work)
4890 {
4891         struct drm_i915_file_private *file_priv =
4892                 container_of(work, typeof(*file_priv), mm.idle_work.work);
4893
4894         atomic_set(&file_priv->rps_wait_boost, false);
4895 }
4896
4897 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4898 {
4899         struct drm_i915_file_private *file_priv;
4900
4901         DRM_DEBUG_DRIVER("\n");
4902
4903         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4904         if (!file_priv)
4905                 return -ENOMEM;
4906
4907         file->driver_priv = file_priv;
4908         file_priv->dev_priv = dev->dev_private;
4909
4910         spin_lock_init(&file_priv->mm.lock);
4911         INIT_LIST_HEAD(&file_priv->mm.request_list);
4912         INIT_DELAYED_WORK(&file_priv->mm.idle_work,
4913                           i915_gem_file_idle_work_handler);
4914
4915         idr_init(&file_priv->context_idr);
4916
4917         return 0;
4918 }
4919
4920 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
4921 {
4922         if (!mutex_is_locked(mutex))
4923                 return false;
4924
4925 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
4926         return mutex->owner == task;
4927 #else
4928         /* Since UP may be pre-empted, we cannot assume that we own the lock */
4929         return false;
4930 #endif
4931 }
4932
4933 static unsigned long
4934 i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
4935 {
4936         struct drm_i915_private *dev_priv =
4937                 container_of(shrinker,
4938                              struct drm_i915_private,
4939                              mm.inactive_shrinker);
4940         struct drm_device *dev = dev_priv->dev;
4941         struct drm_i915_gem_object *obj;
4942         bool unlock = true;
4943         unsigned long count;
4944
4945         if (!mutex_trylock(&dev->struct_mutex)) {
4946                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
4947                         return 0;
4948
4949                 if (dev_priv->mm.shrinker_no_lock_stealing)
4950                         return 0;
4951
4952                 unlock = false;
4953         }
4954
4955         count = 0;
4956         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4957                 if (obj->pages_pin_count == 0)
4958                         count += obj->base.size >> PAGE_SHIFT;
4959
4960         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4961                 if (obj->active)
4962                         continue;
4963
4964                 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
4965                         count += obj->base.size >> PAGE_SHIFT;
4966         }
4967
4968         if (unlock)
4969                 mutex_unlock(&dev->struct_mutex);
4970
4971         return count;
4972 }
4973
4974 /* All the new VM stuff */
4975 unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
4976                                   struct i915_address_space *vm)
4977 {
4978         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
4979         struct i915_vma *vma;
4980
4981         if (vm == &dev_priv->mm.aliasing_ppgtt->base)
4982                 vm = &dev_priv->gtt.base;
4983
4984         BUG_ON(list_empty(&o->vma_list));
4985         list_for_each_entry(vma, &o->vma_list, vma_link) {
4986                 if (vma->vm == vm)
4987                         return vma->node.start;
4988
4989         }
4990         return -1;
4991 }
4992
4993 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4994                         struct i915_address_space *vm)
4995 {
4996         struct i915_vma *vma;
4997
4998         list_for_each_entry(vma, &o->vma_list, vma_link)
4999                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5000                         return true;
5001
5002         return false;
5003 }
5004
5005 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5006 {
5007         struct i915_vma *vma;
5008
5009         list_for_each_entry(vma, &o->vma_list, vma_link)
5010                 if (drm_mm_node_allocated(&vma->node))
5011                         return true;
5012
5013         return false;
5014 }
5015
5016 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5017                                 struct i915_address_space *vm)
5018 {
5019         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5020         struct i915_vma *vma;
5021
5022         if (vm == &dev_priv->mm.aliasing_ppgtt->base)
5023                 vm = &dev_priv->gtt.base;
5024
5025         BUG_ON(list_empty(&o->vma_list));
5026
5027         list_for_each_entry(vma, &o->vma_list, vma_link)
5028                 if (vma->vm == vm)
5029                         return vma->node.size;
5030
5031         return 0;
5032 }
5033
5034 static unsigned long
5035 i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc)
5036 {
5037         struct drm_i915_private *dev_priv =
5038                 container_of(shrinker,
5039                              struct drm_i915_private,
5040                              mm.inactive_shrinker);
5041         struct drm_device *dev = dev_priv->dev;
5042         unsigned long freed;
5043         bool unlock = true;
5044
5045         if (!mutex_trylock(&dev->struct_mutex)) {
5046                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
5047                         return SHRINK_STOP;
5048
5049                 if (dev_priv->mm.shrinker_no_lock_stealing)
5050                         return SHRINK_STOP;
5051
5052                 unlock = false;
5053         }
5054
5055         freed = i915_gem_purge(dev_priv, sc->nr_to_scan);
5056         if (freed < sc->nr_to_scan)
5057                 freed += __i915_gem_shrink(dev_priv,
5058                                            sc->nr_to_scan - freed,
5059                                            false);
5060         if (freed < sc->nr_to_scan)
5061                 freed += i915_gem_shrink_all(dev_priv);
5062
5063         if (unlock)
5064                 mutex_unlock(&dev->struct_mutex);
5065
5066         return freed;
5067 }
5068
5069 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
5070 {
5071         struct i915_vma *vma;
5072
5073         if (WARN_ON(list_empty(&obj->vma_list)))
5074                 return NULL;
5075
5076         vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5077         if (WARN_ON(vma->vm != obj_to_ggtt(obj)))
5078                 return NULL;
5079
5080         return vma;
5081 }