SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drm.h"
  33 #include "i915_drv.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 void
  38 i915_gem_flush(struct drm_device *dev,
  39                uint32_t invalidate_domains,
  40                uint32_t flush_domains)
  41 {
  42         drm_i915_private_t *dev_priv = dev->dev_private;
  43         uint32_t cmd;
  44         RING_LOCALS;
  45
  46 #if WATCH_EXEC
  47         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  48                   invalidate_domains, flush_domains);
  49 #endif
  50         trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
  51                                      invalidate_domains, flush_domains);
  52
  53         if (flush_domains & I915_GEM_DOMAIN_CPU)
  54                 drm_agp_chipset_flush(dev);
  55
  56         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  57                 /*
  58                  * read/write caches:
  59                  *
  60                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  61                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  62                  * also flushed at 2d versus 3d pipeline switches.
  63                  *
  64                  * read-only caches:
  65                  *
  66                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  67                  * MI_READ_FLUSH is set, and is always flushed on 965.
  68                  *
  69                  * I915_GEM_DOMAIN_COMMAND may not exist?
  70                  *
  71                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  72                  * invalidated when MI_EXE_FLUSH is set.
  73                  *
  74                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  75                  * invalidated with every MI_FLUSH.
  76                  *
  77                  * TLBs:
  78                  *
  79                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  80                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  81                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  82                  * are flushed at any MI_FLUSH.
  83                  */
  84
  85                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  86                 if ((invalidate_domains|flush_domains) &
  87                     I915_GEM_DOMAIN_RENDER)
  88                         cmd &= ~MI_NO_WRITE_FLUSH;
  89                 if (!IS_I965G(dev)) {
  90                         /*
  91                          * On the 965, the sampler cache always gets flushed
  92                          * and this bit is reserved.
  93                          */
  94                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  95                                 cmd |= MI_READ_FLUSH;
  96                 }
  97                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  98                         cmd |= MI_EXE_FLUSH;
  99
 100 #if WATCH_EXEC
 101                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 102 #endif
 103                 BEGIN_LP_RING(2);
 104                 OUT_RING(cmd);
 105                 OUT_RING(MI_NOOP);
 106                 ADVANCE_LP_RING();
 107         }
 108
 109 }
 110 #define PIPE_CONTROL_FLUSH(addr)                                        \
 111         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
 112                  PIPE_CONTROL_DEPTH_STALL);                             \
 113         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
 114         OUT_RING(0);                                                    \
 115         OUT_RING(0);                                                    \
 116
 117 /**
 118  * Creates a new sequence number, emitting a write of it to the status page
 119  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 120  *
 121  * Must be called with struct_lock held.
 122  *
 123  * Returned sequence numbers are nonzero on success.
 124  */
 125 uint32_t
 126 i915_ring_add_request(struct drm_device *dev)
 127 {
 128         drm_i915_private_t *dev_priv = dev->dev_private;
 129         uint32_t seqno;
 130         RING_LOCALS;
 131
 132         /* Grab the seqno we're going to make this request be, and bump the
 133          * next (skipping 0 so it can be the reserved no-seqno value).
 134          */
 135         seqno = dev_priv->mm.next_gem_seqno;
 136         dev_priv->mm.next_gem_seqno++;
 137         if (dev_priv->mm.next_gem_seqno == 0)
 138                 dev_priv->mm.next_gem_seqno++;
 139
 140         if (HAS_PIPE_CONTROL(dev)) {
 141                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 142
 143                 /*
 144                  * Workaround qword write incoherence by flushing the
 145                  * PIPE_NOTIFY buffers out to memory before requesting
 146                  * an interrupt.
 147                  */
 148                 BEGIN_LP_RING(32);
 149                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 150                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 151                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 152                 OUT_RING(seqno);
 153                 OUT_RING(0);
 154                 PIPE_CONTROL_FLUSH(scratch_addr);
 155                 scratch_addr += 128; /* write to separate cachelines */
 156                 PIPE_CONTROL_FLUSH(scratch_addr);
 157                 scratch_addr += 128;
 158                 PIPE_CONTROL_FLUSH(scratch_addr);
 159                 scratch_addr += 128;
 160                 PIPE_CONTROL_FLUSH(scratch_addr);
 161                 scratch_addr += 128;
 162                 PIPE_CONTROL_FLUSH(scratch_addr);
 163                 scratch_addr += 128;
 164                 PIPE_CONTROL_FLUSH(scratch_addr);
 165                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 166                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 167                          PIPE_CONTROL_NOTIFY);
 168                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 169                 OUT_RING(seqno);
 170                 OUT_RING(0);
 171                 ADVANCE_LP_RING();
 172         } else {
 173                 BEGIN_LP_RING(4);
 174                 OUT_RING(MI_STORE_DWORD_INDEX);
 175                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 176                 OUT_RING(seqno);
 177
 178                 OUT_RING(MI_USER_INTERRUPT);
 179                 ADVANCE_LP_RING();
 180         }
 181         return seqno;
 182 }
 183
 184 void i915_user_irq_get(struct drm_device *dev)
 185 {
 186         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 187         unsigned long irqflags;
 188
 189         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 190         if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)) {
 191                 if (HAS_PCH_SPLIT(dev))
 192                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 193                 else
 194                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 195         }
 196         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 197 }
 198
 199 void i915_user_irq_put(struct drm_device *dev)
 200 {
 201         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 202         unsigned long irqflags;
 203
 204         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 205         BUG_ON(dev->irq_enabled && dev_priv->user_irq_refcount <= 0);
 206         if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
 207                 if (HAS_PCH_SPLIT(dev))
 208                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 209                 else
 210                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 211         }
 212         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 213 }
 214
 215 /** Dispatch a batchbuffer to the ring
 216  */
 217 int
 218 i915_dispatch_gem_execbuffer(struct drm_device *dev,
 219                               struct drm_i915_gem_execbuffer2 *exec,
 220                               struct drm_clip_rect *cliprects,
 221                               uint64_t exec_offset)
 222 {
 223         drm_i915_private_t *dev_priv = dev->dev_private;
 224         int nbox = exec->num_cliprects;
 225         int i = 0, count;
 226         uint32_t exec_start, exec_len;
 227         RING_LOCALS;
 228
 229         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 230         exec_len = (uint32_t) exec->batch_len;
 231
 232         trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
 233
 234         count = nbox ? nbox : 1;
 235
 236         for (i = 0; i < count; i++) {
 237                 if (i < nbox) {
 238                         int ret = i915_emit_box(dev, cliprects, i,
 239                                                 exec->DR1, exec->DR4);
 240                         if (ret)
 241                                 return ret;
 242                 }
 243
 244                 if (IS_I830(dev) || IS_845G(dev)) {
 245                         BEGIN_LP_RING(4);
 246                         OUT_RING(MI_BATCH_BUFFER);
 247                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
 248                         OUT_RING(exec_start + exec_len - 4);
 249                         OUT_RING(0);
 250                         ADVANCE_LP_RING();
 251                 } else {
 252                         BEGIN_LP_RING(2);
 253                         if (IS_I965G(dev)) {
 254                                 OUT_RING(MI_BATCH_BUFFER_START |
 255                                          (2 << 6) |
 256                                          MI_BATCH_NON_SECURE_I965);
 257                                 OUT_RING(exec_start);
 258                         } else {
 259                                 OUT_RING(MI_BATCH_BUFFER_START |
 260                                          (2 << 6));
 261                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
 262                         }
 263                         ADVANCE_LP_RING();
 264                 }
 265         }
 266
 267         /* XXX breadcrumb */
 268         return 0;
 269 }
 270
 271 static void
 272 i915_gem_cleanup_hws(struct drm_device *dev)
 273 {
 274         drm_i915_private_t *dev_priv = dev->dev_private;
 275         struct drm_gem_object *obj;
 276         struct drm_i915_gem_object *obj_priv;
 277
 278         if (dev_priv->hws_obj == NULL)
 279                 return;
 280
 281         obj = dev_priv->hws_obj;
 282         obj_priv = to_intel_bo(obj);
 283
 284         kunmap(obj_priv->pages[0]);
 285         i915_gem_object_unpin(obj);
 286         drm_gem_object_unreference(obj);
 287         dev_priv->hws_obj = NULL;
 288
 289         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 290         dev_priv->hw_status_page = NULL;
 291
 292         if (HAS_PIPE_CONTROL(dev))
 293                 i915_gem_cleanup_pipe_control(dev);
 294
 295         /* Write high address into HWS_PGA when disabling. */
 296         I915_WRITE(HWS_PGA, 0x1ffff000);
 297 }
 298
 299 static int
 300 i915_gem_init_hws(struct drm_device *dev)
 301 {
 302         drm_i915_private_t *dev_priv = dev->dev_private;
 303         struct drm_gem_object *obj;
 304         struct drm_i915_gem_object *obj_priv;
 305         int ret;
 306
 307         /* If we need a physical address for the status page, it's already
 308          * initialized at driver load time.
 309          */
 310         if (!I915_NEED_GFX_HWS(dev))
 311                 return 0;
 312
 313         obj = i915_gem_alloc_object(dev, 4096);
 314         if (obj == NULL) {
 315                 DRM_ERROR("Failed to allocate status page\n");
 316                 ret = -ENOMEM;
 317                 goto err;
 318         }
 319         obj_priv = to_intel_bo(obj);
 320         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 321
 322         ret = i915_gem_object_pin(obj, 4096);
 323         if (ret != 0) {
 324                 drm_gem_object_unreference(obj);
 325                 goto err_unref;
 326         }
 327
 328         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
 329
 330         dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
 331         if (dev_priv->hw_status_page == NULL) {
 332                 DRM_ERROR("Failed to map status page.\n");
 333                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 334                 ret = -EINVAL;
 335                 goto err_unpin;
 336         }
 337
 338         if (HAS_PIPE_CONTROL(dev)) {
 339                 ret = i915_gem_init_pipe_control(dev);
 340                 if (ret)
 341                         goto err_unpin;
 342         }
 343
 344         dev_priv->hws_obj = obj;
 345         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
 346         if (IS_GEN6(dev)) {
 347                 I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
 348                 I915_READ(HWS_PGA_GEN6); /* posting read */
 349         } else {
 350                 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
 351                 I915_READ(HWS_PGA); /* posting read */
 352         }
 353         DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
 354
 355         return 0;
 356
 357 err_unpin:
 358         i915_gem_object_unpin(obj);
 359 err_unref:
 360         drm_gem_object_unreference(obj);
 361 err:
 362         return 0;
 363 }
 364
 365 int
 366 i915_gem_init_ringbuffer(struct drm_device *dev)
 367 {
 368         drm_i915_private_t *dev_priv = dev->dev_private;
 369         struct drm_gem_object *obj;
 370         struct drm_i915_gem_object *obj_priv;
 371         drm_i915_ring_buffer_t *ring = &dev_priv->render_ring;
 372         int ret;
 373         u32 head;
 374
 375         ret = i915_gem_init_hws(dev);
 376         if (ret != 0)
 377                 return ret;
 378
 379         obj = i915_gem_alloc_object(dev, 128 * 1024);
 380         if (obj == NULL) {
 381                 DRM_ERROR("Failed to allocate ringbuffer\n");
 382                 i915_gem_cleanup_hws(dev);
 383                 return -ENOMEM;
 384         }
 385         obj_priv = to_intel_bo(obj);
 386
 387         ret = i915_gem_object_pin(obj, 4096);
 388         if (ret != 0) {
 389                 drm_gem_object_unreference(obj);
 390                 i915_gem_cleanup_hws(dev);
 391                 return ret;
 392         }
 393
 394         /* Set up the kernel mapping for the ring. */
 395         ring->Size = obj->size;
 396
 397         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 398         ring->map.size = obj->size;
 399         ring->map.type = 0;
 400         ring->map.flags = 0;
 401         ring->map.mtrr = 0;
 402
 403         drm_core_ioremap_wc(&ring->map, dev);
 404         if (ring->map.handle == NULL) {
 405                 DRM_ERROR("Failed to map ringbuffer.\n");
 406                 memset(&dev_priv->render_ring, 0, sizeof(dev_priv->render_ring));
 407                 i915_gem_object_unpin(obj);
 408                 drm_gem_object_unreference(obj);
 409                 i915_gem_cleanup_hws(dev);
 410                 return -EINVAL;
 411         }
 412         ring->ring_obj = obj;
 413         ring->virtual_start = ring->map.handle;
 414
 415         /* Stop the ring if it's running. */
 416         I915_WRITE(PRB0_CTL, 0);
 417         I915_WRITE(PRB0_TAIL, 0);
 418         I915_WRITE(PRB0_HEAD, 0);
 419
 420         /* Initialize the ring. */
 421         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
 422         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 423
 424         /* G45 ring initialization fails to reset head to zero */
 425         if (head != 0) {
 426                 DRM_ERROR("Ring head not reset to zero "
 427                           "ctl %08x head %08x tail %08x start %08x\n",
 428                           I915_READ(PRB0_CTL),
 429                           I915_READ(PRB0_HEAD),
 430                           I915_READ(PRB0_TAIL),
 431                           I915_READ(PRB0_START));
 432                 I915_WRITE(PRB0_HEAD, 0);
 433
 434                 DRM_ERROR("Ring head forced to zero "
 435                           "ctl %08x head %08x tail %08x start %08x\n",
 436                           I915_READ(PRB0_CTL),
 437                           I915_READ(PRB0_HEAD),
 438                           I915_READ(PRB0_TAIL),
 439                           I915_READ(PRB0_START));
 440         }
 441
 442         I915_WRITE(PRB0_CTL,
 443                    ((obj->size - 4096) & RING_NR_PAGES) |
 444                    RING_NO_REPORT |
 445                    RING_VALID);
 446
 447         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 448
 449         /* If the head is still not zero, the ring is dead */
 450         if (head != 0) {
 451                 DRM_ERROR("Ring initialization failed "
 452                           "ctl %08x head %08x tail %08x start %08x\n",
 453                           I915_READ(PRB0_CTL),
 454                           I915_READ(PRB0_HEAD),
 455                           I915_READ(PRB0_TAIL),
 456                           I915_READ(PRB0_START));
 457                 return -EIO;
 458         }
 459
 460         /* Update our cache of the ring state */
 461         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 462                 i915_kernel_lost_context(dev);
 463         else {
 464                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 465                 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
 466                 ring->space = ring->head - (ring->tail + 8);
 467                 if (ring->space < 0)
 468                         ring->space += ring->Size;
 469         }
 470
 471         if (IS_I9XX(dev) && !IS_GEN3(dev)) {
 472                 I915_WRITE(MI_MODE,
 473                            (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
 474         }
 475
 476         return 0;
 477 }
 478
 479 void
 480 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 481 {
 482         drm_i915_private_t *dev_priv = dev->dev_private;
 483
 484         if (dev_priv->render_ring.ring_obj == NULL)
 485                 return;
 486
 487         drm_core_ioremapfree(&dev_priv->render_ring.map, dev);
 488
 489         i915_gem_object_unpin(dev_priv->render_ring.ring_obj);
 490         drm_gem_object_unreference(dev_priv->render_ring.ring_obj);
 491         dev_priv->render_ring.ring_obj = NULL;
 492         memset(&dev_priv->render_ring, 0, sizeof(dev_priv->render_ring));
 493
 494         i915_gem_cleanup_hws(dev);
 495 }
 496
 497 /* As a ringbuffer is only allowed to wrap between instructions, fill
 498  * the tail with NOOPs.
 499  */
 500 int i915_wrap_ring(struct drm_device *dev)
 501 {
 502         drm_i915_private_t *dev_priv = dev->dev_private;
 503         volatile unsigned int *virt;
 504         int rem;
 505
 506         rem = dev_priv->render_ring.Size - dev_priv->render_ring.tail;
 507         if (dev_priv->render_ring.space < rem) {
 508                 int ret = i915_wait_ring(dev, rem, __func__);
 509                 if (ret)
 510                         return ret;
 511         }
 512         dev_priv->render_ring.space -= rem;
 513
 514         virt = (unsigned int *)
 515                 (dev_priv->render_ring.virtual_start + dev_priv->render_ring.tail);
 516         rem /= 4;
 517         while (rem--)
 518                 *virt++ = MI_NOOP;
 519
 520         dev_priv->render_ring.tail = 0;
 521
 522         return 0;
 523 }
 524
 525 int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
 526 {
 527         drm_i915_private_t *dev_priv = dev->dev_private;
 528         drm_i915_ring_buffer_t *ring = &(dev_priv->render_ring);
 529         u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
 530         u32 last_acthd = I915_READ(acthd_reg);
 531         u32 acthd;
 532         u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 533         int i;
 534
 535         trace_i915_ring_wait_begin (dev);
 536
 537         for (i = 0; i < 100000; i++) {
 538                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 539                 acthd = I915_READ(acthd_reg);
 540                 ring->space = ring->head - (ring->tail + 8);
 541                 if (ring->space < 0)
 542                         ring->space += ring->Size;
 543                 if (ring->space >= n) {
 544                         trace_i915_ring_wait_end (dev);
 545                         return 0;
 546                 }
 547
 548                 if (dev->primary->master) {
 549                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 550                         if (master_priv->sarea_priv)
 551                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 552                 }
 553
 554
 555                 if (ring->head != last_head)
 556                         i = 0;
 557                 if (acthd != last_acthd)
 558                         i = 0;
 559
 560                 last_head = ring->head;
 561                 last_acthd = acthd;
 562                 msleep_interruptible(10);
 563
 564         }
 565
 566         trace_i915_ring_wait_end (dev);
 567         return -EBUSY;
 568 }