drivers/gpu/drm/msm/msm_gpu.c

   1 /*
   2  * Copyright (C) 2013 Red Hat
   3  * Author: Rob Clark <robdclark@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "msm_gpu.h"
  19 #include "msm_gem.h"
  20 #include "msm_mmu.h"
  21
  22
  23 /*
  24  * Power Management:
  25  */
  26
  27 #ifdef CONFIG_MSM_BUS_SCALING
  28 #include <mach/board.h>
  29 static void bs_init(struct msm_gpu *gpu)
  30 {
  31         if (gpu->bus_scale_table) {
  32                 gpu->bsc = msm_bus_scale_register_client(gpu->bus_scale_table);
  33                 DBG("bus scale client: %08x", gpu->bsc);
  34         }
  35 }
  36
  37 static void bs_fini(struct msm_gpu *gpu)
  38 {
  39         if (gpu->bsc) {
  40                 msm_bus_scale_unregister_client(gpu->bsc);
  41                 gpu->bsc = 0;
  42         }
  43 }
  44
  45 static void bs_set(struct msm_gpu *gpu, int idx)
  46 {
  47         if (gpu->bsc) {
  48                 DBG("set bus scaling: %d", idx);
  49                 msm_bus_scale_client_update_request(gpu->bsc, idx);
  50         }
  51 }
  52 #else
  53 static void bs_init(struct msm_gpu *gpu) {}
  54 static void bs_fini(struct msm_gpu *gpu) {}
  55 static void bs_set(struct msm_gpu *gpu, int idx) {}
  56 #endif
  57
  58 static int enable_pwrrail(struct msm_gpu *gpu)
  59 {
  60         struct drm_device *dev = gpu->dev;
  61         int ret = 0;
  62
  63         if (gpu->gpu_reg) {
  64                 ret = regulator_enable(gpu->gpu_reg);
  65                 if (ret) {
  66                         dev_err(dev->dev, "failed to enable 'gpu_reg': %d\n", ret);
  67                         return ret;
  68                 }
  69         }
  70
  71         if (gpu->gpu_cx) {
  72                 ret = regulator_enable(gpu->gpu_cx);
  73                 if (ret) {
  74                         dev_err(dev->dev, "failed to enable 'gpu_cx': %d\n", ret);
  75                         return ret;
  76                 }
  77         }
  78
  79         return 0;
  80 }
  81
  82 static int disable_pwrrail(struct msm_gpu *gpu)
  83 {
  84         if (gpu->gpu_cx)
  85                 regulator_disable(gpu->gpu_cx);
  86         if (gpu->gpu_reg)
  87                 regulator_disable(gpu->gpu_reg);
  88         return 0;
  89 }
  90
  91 static int enable_clk(struct msm_gpu *gpu)
  92 {
  93         struct clk *rate_clk = NULL;
  94         int i;
  95
  96         /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
  97         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
  98                 if (gpu->grp_clks[i]) {
  99                         clk_prepare(gpu->grp_clks[i]);
 100                         rate_clk = gpu->grp_clks[i];
 101                 }
 102         }
 103
 104         if (rate_clk && gpu->fast_rate)
 105                 clk_set_rate(rate_clk, gpu->fast_rate);
 106
 107         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
 108                 if (gpu->grp_clks[i])
 109                         clk_enable(gpu->grp_clks[i]);
 110
 111         return 0;
 112 }
 113
 114 static int disable_clk(struct msm_gpu *gpu)
 115 {
 116         struct clk *rate_clk = NULL;
 117         int i;
 118
 119         /* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
 120         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
 121                 if (gpu->grp_clks[i]) {
 122                         clk_disable(gpu->grp_clks[i]);
 123                         rate_clk = gpu->grp_clks[i];
 124                 }
 125         }
 126
 127         if (rate_clk && gpu->slow_rate)
 128                 clk_set_rate(rate_clk, gpu->slow_rate);
 129
 130         for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
 131                 if (gpu->grp_clks[i])
 132                         clk_unprepare(gpu->grp_clks[i]);
 133
 134         return 0;
 135 }
 136
 137 static int enable_axi(struct msm_gpu *gpu)
 138 {
 139         if (gpu->ebi1_clk)
 140                 clk_prepare_enable(gpu->ebi1_clk);
 141         if (gpu->bus_freq)
 142                 bs_set(gpu, gpu->bus_freq);
 143         return 0;
 144 }
 145
 146 static int disable_axi(struct msm_gpu *gpu)
 147 {
 148         if (gpu->ebi1_clk)
 149                 clk_disable_unprepare(gpu->ebi1_clk);
 150         if (gpu->bus_freq)
 151                 bs_set(gpu, 0);
 152         return 0;
 153 }
 154
 155 int msm_gpu_pm_resume(struct msm_gpu *gpu)
 156 {
 157         int ret;
 158
 159         DBG("%s", gpu->name);
 160
 161         ret = enable_pwrrail(gpu);
 162         if (ret)
 163                 return ret;
 164
 165         ret = enable_clk(gpu);
 166         if (ret)
 167                 return ret;
 168
 169         ret = enable_axi(gpu);
 170         if (ret)
 171                 return ret;
 172
 173         return 0;
 174 }
 175
 176 int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 177 {
 178         int ret;
 179
 180         DBG("%s", gpu->name);
 181
 182         ret = disable_axi(gpu);
 183         if (ret)
 184                 return ret;
 185
 186         ret = disable_clk(gpu);
 187         if (ret)
 188                 return ret;
 189
 190         ret = disable_pwrrail(gpu);
 191         if (ret)
 192                 return ret;
 193
 194         return 0;
 195 }
 196
 197 /*
 198  * Hangcheck detection for locked gpu:
 199  */
 200
 201 static void recover_worker(struct work_struct *work)
 202 {
 203         struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);
 204         struct drm_device *dev = gpu->dev;
 205
 206         dev_err(dev->dev, "%s: hangcheck recover!\n", gpu->name);
 207
 208         mutex_lock(&dev->struct_mutex);
 209         gpu->funcs->recover(gpu);
 210         mutex_unlock(&dev->struct_mutex);
 211
 212         msm_gpu_retire(gpu);
 213 }
 214
 215 static void hangcheck_timer_reset(struct msm_gpu *gpu)
 216 {
 217         DBG("%s", gpu->name);
 218         mod_timer(&gpu->hangcheck_timer,
 219                         round_jiffies_up(jiffies + DRM_MSM_HANGCHECK_JIFFIES));
 220 }
 221
 222 static void hangcheck_handler(unsigned long data)
 223 {
 224         struct msm_gpu *gpu = (struct msm_gpu *)data;
 225         struct drm_device *dev = gpu->dev;
 226         struct msm_drm_private *priv = dev->dev_private;
 227         uint32_t fence = gpu->funcs->last_fence(gpu);
 228
 229         if (fence != gpu->hangcheck_fence) {
 230                 /* some progress has been made.. ya! */
 231                 gpu->hangcheck_fence = fence;
 232         } else if (fence < gpu->submitted_fence) {
 233                 /* no progress and not done.. hung! */
 234                 gpu->hangcheck_fence = fence;
 235                 dev_err(dev->dev, "%s: hangcheck detected gpu lockup!\n",
 236                                 gpu->name);
 237                 dev_err(dev->dev, "%s:     completed fence: %u\n",
 238                                 gpu->name, fence);
 239                 dev_err(dev->dev, "%s:     submitted fence: %u\n",
 240                                 gpu->name, gpu->submitted_fence);
 241                 queue_work(priv->wq, &gpu->recover_work);
 242         }
 243
 244         /* if still more pending work, reset the hangcheck timer: */
 245         if (gpu->submitted_fence > gpu->hangcheck_fence)
 246                 hangcheck_timer_reset(gpu);
 247
 248         /* workaround for missing irq: */
 249         queue_work(priv->wq, &gpu->retire_work);
 250 }
 251
 252 /*
 253  * Cmdstream submission/retirement:
 254  */
 255
 256 static void retire_worker(struct work_struct *work)
 257 {
 258         struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work);
 259         struct drm_device *dev = gpu->dev;
 260         uint32_t fence = gpu->funcs->last_fence(gpu);
 261
 262         msm_update_fence(gpu->dev, fence);
 263
 264         mutex_lock(&dev->struct_mutex);
 265
 266         while (!list_empty(&gpu->active_list)) {
 267                 struct msm_gem_object *obj;
 268
 269                 obj = list_first_entry(&gpu->active_list,
 270                                 struct msm_gem_object, mm_list);
 271
 272                 if ((obj->read_fence <= fence) &&
 273                                 (obj->write_fence <= fence)) {
 274                         /* move to inactive: */
 275                         msm_gem_move_to_inactive(&obj->base);
 276                         msm_gem_put_iova(&obj->base, gpu->id);
 277                         drm_gem_object_unreference(&obj->base);
 278                 } else {
 279                         break;
 280                 }
 281         }
 282
 283         mutex_unlock(&dev->struct_mutex);
 284 }
 285
 286 /* call from irq handler to schedule work to retire bo's */
 287 void msm_gpu_retire(struct msm_gpu *gpu)
 288 {
 289         struct msm_drm_private *priv = gpu->dev->dev_private;
 290         queue_work(priv->wq, &gpu->retire_work);
 291 }
 292
 293 /* add bo's to gpu's ring, and kick gpu: */
 294 int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 295                 struct msm_file_private *ctx)
 296 {
 297         struct drm_device *dev = gpu->dev;
 298         struct msm_drm_private *priv = dev->dev_private;
 299         int i, ret;
 300
 301         submit->fence = ++priv->next_fence;
 302
 303         gpu->submitted_fence = submit->fence;
 304
 305         ret = gpu->funcs->submit(gpu, submit, ctx);
 306         priv->lastctx = ctx;
 307
 308         for (i = 0; i < submit->nr_bos; i++) {
 309                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 310
 311                 /* can't happen yet.. but when we add 2d support we'll have
 312                  * to deal w/ cross-ring synchronization:
 313                  */
 314                 WARN_ON(is_active(msm_obj) && (msm_obj->gpu != gpu));
 315
 316                 if (!is_active(msm_obj)) {
 317                         uint32_t iova;
 318
 319                         /* ring takes a reference to the bo and iova: */
 320                         drm_gem_object_reference(&msm_obj->base);
 321                         msm_gem_get_iova_locked(&msm_obj->base,
 322                                         submit->gpu->id, &iova);
 323                 }
 324
 325                 if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
 326                         msm_gem_move_to_active(&msm_obj->base, gpu, false, submit->fence);
 327
 328                 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
 329                         msm_gem_move_to_active(&msm_obj->base, gpu, true, submit->fence);
 330         }
 331         hangcheck_timer_reset(gpu);
 332
 333         return ret;
 334 }
 335
 336 /*
 337  * Init/Cleanup:
 338  */
 339
 340 static irqreturn_t irq_handler(int irq, void *data)
 341 {
 342         struct msm_gpu *gpu = data;
 343         return gpu->funcs->irq(gpu);
 344 }
 345
 346 static const char *clk_names[] = {
 347                 "src_clk", "core_clk", "iface_clk", "mem_clk", "mem_iface_clk",
 348 };
 349
 350 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 351                 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
 352                 const char *name, const char *ioname, const char *irqname, int ringsz)
 353 {
 354         struct iommu_domain *iommu;
 355         int i, ret;
 356
 357         gpu->dev = drm;
 358         gpu->funcs = funcs;
 359         gpu->name = name;
 360
 361         INIT_LIST_HEAD(&gpu->active_list);
 362         INIT_WORK(&gpu->retire_work, retire_worker);
 363         INIT_WORK(&gpu->recover_work, recover_worker);
 364
 365         setup_timer(&gpu->hangcheck_timer, hangcheck_handler,
 366                         (unsigned long)gpu);
 367
 368         BUG_ON(ARRAY_SIZE(clk_names) != ARRAY_SIZE(gpu->grp_clks));
 369
 370         /* Map registers: */
 371         gpu->mmio = msm_ioremap(pdev, ioname, name);
 372         if (IS_ERR(gpu->mmio)) {
 373                 ret = PTR_ERR(gpu->mmio);
 374                 goto fail;
 375         }
 376
 377         /* Get Interrupt: */
 378         gpu->irq = platform_get_irq_byname(pdev, irqname);
 379         if (gpu->irq < 0) {
 380                 ret = gpu->irq;
 381                 dev_err(drm->dev, "failed to get irq: %d\n", ret);
 382                 goto fail;
 383         }
 384
 385         ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler,
 386                         IRQF_TRIGGER_HIGH, gpu->name, gpu);
 387         if (ret) {
 388                 dev_err(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret);
 389                 goto fail;
 390         }
 391
 392         /* Acquire clocks: */
 393         for (i = 0; i < ARRAY_SIZE(clk_names); i++) {
 394                 gpu->grp_clks[i] = devm_clk_get(&pdev->dev, clk_names[i]);
 395                 DBG("grp_clks[%s]: %p", clk_names[i], gpu->grp_clks[i]);
 396                 if (IS_ERR(gpu->grp_clks[i]))
 397                         gpu->grp_clks[i] = NULL;
 398         }
 399
 400         gpu->ebi1_clk = devm_clk_get(&pdev->dev, "bus_clk");
 401         DBG("ebi1_clk: %p", gpu->ebi1_clk);
 402         if (IS_ERR(gpu->ebi1_clk))
 403                 gpu->ebi1_clk = NULL;
 404
 405         /* Acquire regulators: */
 406         gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd");
 407         DBG("gpu_reg: %p", gpu->gpu_reg);
 408         if (IS_ERR(gpu->gpu_reg))
 409                 gpu->gpu_reg = NULL;
 410
 411         gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx");
 412         DBG("gpu_cx: %p", gpu->gpu_cx);
 413         if (IS_ERR(gpu->gpu_cx))
 414                 gpu->gpu_cx = NULL;
 415
 416         /* Setup IOMMU.. eventually we will (I think) do this once per context
 417          * and have separate page tables per context.  For now, to keep things
 418          * simple and to get something working, just use a single address space:
 419          */
 420         iommu = iommu_domain_alloc(&platform_bus_type);
 421         if (iommu) {
 422                 dev_info(drm->dev, "%s: using IOMMU\n", name);
 423                 gpu->mmu = msm_iommu_new(drm, iommu);
 424         } else {
 425                 dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
 426         }
 427         gpu->id = msm_register_mmu(drm, gpu->mmu);
 428
 429         /* Create ringbuffer: */
 430         gpu->rb = msm_ringbuffer_new(gpu, ringsz);
 431         if (IS_ERR(gpu->rb)) {
 432                 ret = PTR_ERR(gpu->rb);
 433                 gpu->rb = NULL;
 434                 dev_err(drm->dev, "could not create ringbuffer: %d\n", ret);
 435                 goto fail;
 436         }
 437
 438         ret = msm_gem_get_iova_locked(gpu->rb->bo, gpu->id, &gpu->rb_iova);
 439         if (ret) {
 440                 gpu->rb_iova = 0;
 441                 dev_err(drm->dev, "could not map ringbuffer: %d\n", ret);
 442                 goto fail;
 443         }
 444
 445         bs_init(gpu);
 446
 447         return 0;
 448
 449 fail:
 450         return ret;
 451 }
 452
 453 void msm_gpu_cleanup(struct msm_gpu *gpu)
 454 {
 455         DBG("%s", gpu->name);
 456
 457         WARN_ON(!list_empty(&gpu->active_list));
 458
 459         bs_fini(gpu);
 460
 461         if (gpu->rb) {
 462                 if (gpu->rb_iova)
 463                         msm_gem_put_iova(gpu->rb->bo, gpu->id);
 464                 msm_ringbuffer_destroy(gpu->rb);
 465         }
 466
 467         if (gpu->mmu)
 468                 gpu->mmu->funcs->destroy(gpu->mmu);
 469 }