2 * Copyright © 2008 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
32 #include <linux/swap.h>
34 #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
37 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
38 uint32_t read_domains,
39 uint32_t write_domain);
40 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
41 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
42 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
43 static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
45 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
47 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
50 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
51 static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
52 static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
53 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
56 i915_gem_cleanup_ringbuffer(struct drm_device *dev);
59 i915_gem_init_ioctl(struct drm_device *dev, void *data,
60 struct drm_file *file_priv)
62 drm_i915_private_t *dev_priv = dev->dev_private;
63 struct drm_i915_gem_init *args = data;
65 mutex_lock(&dev->struct_mutex);
67 if (args->gtt_start >= args->gtt_end ||
68 (args->gtt_start & (PAGE_SIZE - 1)) != 0 ||
69 (args->gtt_end & (PAGE_SIZE - 1)) != 0) {
70 mutex_unlock(&dev->struct_mutex);
74 drm_mm_init(&dev_priv->mm.gtt_space, args->gtt_start,
75 args->gtt_end - args->gtt_start);
77 dev->gtt_total = (uint32_t) (args->gtt_end - args->gtt_start);
79 mutex_unlock(&dev->struct_mutex);
85 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
86 struct drm_file *file_priv)
88 struct drm_i915_gem_get_aperture *args = data;
90 if (!(dev->driver->driver_features & DRIVER_GEM))
93 args->aper_size = dev->gtt_total;
94 args->aper_available_size = (args->aper_size -
95 atomic_read(&dev->pin_memory));
102 * Creates a new mm object and returns a handle to it.
105 i915_gem_create_ioctl(struct drm_device *dev, void *data,
106 struct drm_file *file_priv)
108 struct drm_i915_gem_create *args = data;
109 struct drm_gem_object *obj;
112 args->size = roundup(args->size, PAGE_SIZE);
114 /* Allocate the new object */
115 obj = drm_gem_object_alloc(dev, args->size);
119 ret = drm_gem_handle_create(file_priv, obj, &handle);
120 mutex_lock(&dev->struct_mutex);
121 drm_gem_object_handle_unreference(obj);
122 mutex_unlock(&dev->struct_mutex);
127 args->handle = handle;
133 * Reads data from the object referenced by handle.
135 * On error, the contents of *data are undefined.
138 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
139 struct drm_file *file_priv)
141 struct drm_i915_gem_pread *args = data;
142 struct drm_gem_object *obj;
143 struct drm_i915_gem_object *obj_priv;
148 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
151 obj_priv = obj->driver_private;
153 /* Bounds check source.
155 * XXX: This could use review for overflow issues...
157 if (args->offset > obj->size || args->size > obj->size ||
158 args->offset + args->size > obj->size) {
159 drm_gem_object_unreference(obj);
163 mutex_lock(&dev->struct_mutex);
165 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
168 drm_gem_object_unreference(obj);
169 mutex_unlock(&dev->struct_mutex);
173 offset = args->offset;
175 read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr,
176 args->size, &offset);
177 if (read != args->size) {
178 drm_gem_object_unreference(obj);
179 mutex_unlock(&dev->struct_mutex);
186 drm_gem_object_unreference(obj);
187 mutex_unlock(&dev->struct_mutex);
192 /* This is the fast write path which cannot handle
193 * page faults in the source data
197 fast_user_write(struct io_mapping *mapping,
198 loff_t page_base, int page_offset,
199 char __user *user_data,
203 unsigned long unwritten;
205 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
206 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
208 io_mapping_unmap_atomic(vaddr_atomic);
214 /* Here's the write path which can sleep for
219 slow_user_write(struct io_mapping *mapping,
220 loff_t page_base, int page_offset,
221 char __user *user_data,
225 unsigned long unwritten;
227 vaddr = io_mapping_map_wc(mapping, page_base);
230 unwritten = __copy_from_user(vaddr + page_offset,
232 io_mapping_unmap(vaddr);
239 i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
240 struct drm_i915_gem_pwrite *args,
241 struct drm_file *file_priv)
243 struct drm_i915_gem_object *obj_priv = obj->driver_private;
244 drm_i915_private_t *dev_priv = dev->dev_private;
246 loff_t offset, page_base;
247 char __user *user_data;
248 int page_offset, page_length;
251 user_data = (char __user *) (uintptr_t) args->data_ptr;
253 if (!access_ok(VERIFY_READ, user_data, remain))
257 mutex_lock(&dev->struct_mutex);
258 ret = i915_gem_object_pin(obj, 0);
260 mutex_unlock(&dev->struct_mutex);
263 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
267 obj_priv = obj->driver_private;
268 offset = obj_priv->gtt_offset + args->offset;
272 /* Operation in this page
274 * page_base = page offset within aperture
275 * page_offset = offset within page
276 * page_length = bytes to copy for this page
278 page_base = (offset & ~(PAGE_SIZE-1));
279 page_offset = offset & (PAGE_SIZE-1);
280 page_length = remain;
281 if ((page_offset + remain) > PAGE_SIZE)
282 page_length = PAGE_SIZE - page_offset;
284 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
285 page_offset, user_data, page_length);
287 /* If we get a fault while copying data, then (presumably) our
288 * source page isn't available. In this case, use the
289 * non-atomic function
292 ret = slow_user_write (dev_priv->mm.gtt_mapping,
293 page_base, page_offset,
294 user_data, page_length);
299 remain -= page_length;
300 user_data += page_length;
301 offset += page_length;
305 i915_gem_object_unpin(obj);
306 mutex_unlock(&dev->struct_mutex);
312 i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
313 struct drm_i915_gem_pwrite *args,
314 struct drm_file *file_priv)
320 mutex_lock(&dev->struct_mutex);
322 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
324 mutex_unlock(&dev->struct_mutex);
328 offset = args->offset;
330 written = vfs_write(obj->filp,
331 (char __user *)(uintptr_t) args->data_ptr,
332 args->size, &offset);
333 if (written != args->size) {
334 mutex_unlock(&dev->struct_mutex);
341 mutex_unlock(&dev->struct_mutex);
347 * Writes data to the object referenced by handle.
349 * On error, the contents of the buffer that were to be modified are undefined.
352 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
353 struct drm_file *file_priv)
355 struct drm_i915_gem_pwrite *args = data;
356 struct drm_gem_object *obj;
357 struct drm_i915_gem_object *obj_priv;
360 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
363 obj_priv = obj->driver_private;
365 /* Bounds check destination.
367 * XXX: This could use review for overflow issues...
369 if (args->offset > obj->size || args->size > obj->size ||
370 args->offset + args->size > obj->size) {
371 drm_gem_object_unreference(obj);
375 /* We can only do the GTT pwrite on untiled buffers, as otherwise
376 * it would end up going through the fenced access, and we'll get
377 * different detiling behavior between reading and writing.
378 * pread/pwrite currently are reading and writing from the CPU
379 * perspective, requiring manual detiling by the client.
381 if (obj_priv->tiling_mode == I915_TILING_NONE &&
383 ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv);
385 ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv);
389 DRM_INFO("pwrite failed %d\n", ret);
392 drm_gem_object_unreference(obj);
398 * Called when user space prepares to use an object with the CPU, either
399 * through the mmap ioctl's mapping or a GTT mapping.
402 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
403 struct drm_file *file_priv)
405 struct drm_i915_gem_set_domain *args = data;
406 struct drm_gem_object *obj;
407 uint32_t read_domains = args->read_domains;
408 uint32_t write_domain = args->write_domain;
411 if (!(dev->driver->driver_features & DRIVER_GEM))
414 /* Only handle setting domains to types used by the CPU. */
415 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
418 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
421 /* Having something in the write domain implies it's in the read
422 * domain, and only that read domain. Enforce that in the request.
424 if (write_domain != 0 && read_domains != write_domain)
427 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
431 mutex_lock(&dev->struct_mutex);
433 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
434 obj, obj->size, read_domains, write_domain);
436 if (read_domains & I915_GEM_DOMAIN_GTT) {
437 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
439 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
442 drm_gem_object_unreference(obj);
443 mutex_unlock(&dev->struct_mutex);
448 * Called when user space has done writes to this buffer
451 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
452 struct drm_file *file_priv)
454 struct drm_i915_gem_sw_finish *args = data;
455 struct drm_gem_object *obj;
456 struct drm_i915_gem_object *obj_priv;
459 if (!(dev->driver->driver_features & DRIVER_GEM))
462 mutex_lock(&dev->struct_mutex);
463 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
465 mutex_unlock(&dev->struct_mutex);
470 DRM_INFO("%s: sw_finish %d (%p %d)\n",
471 __func__, args->handle, obj, obj->size);
473 obj_priv = obj->driver_private;
475 /* Pinned buffers may be scanout, so flush the cache */
476 if (obj_priv->pin_count)
477 i915_gem_object_flush_cpu_write_domain(obj);
479 drm_gem_object_unreference(obj);
480 mutex_unlock(&dev->struct_mutex);
485 * Maps the contents of an object, returning the address it is mapped
488 * While the mapping holds a reference on the contents of the object, it doesn't
489 * imply a ref on the object itself.
492 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
493 struct drm_file *file_priv)
495 struct drm_i915_gem_mmap *args = data;
496 struct drm_gem_object *obj;
500 if (!(dev->driver->driver_features & DRIVER_GEM))
503 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
507 offset = args->offset;
509 down_write(¤t->mm->mmap_sem);
510 addr = do_mmap(obj->filp, 0, args->size,
511 PROT_READ | PROT_WRITE, MAP_SHARED,
513 up_write(¤t->mm->mmap_sem);
514 mutex_lock(&dev->struct_mutex);
515 drm_gem_object_unreference(obj);
516 mutex_unlock(&dev->struct_mutex);
517 if (IS_ERR((void *)addr))
520 args->addr_ptr = (uint64_t) addr;
526 i915_gem_object_free_page_list(struct drm_gem_object *obj)
528 struct drm_i915_gem_object *obj_priv = obj->driver_private;
529 int page_count = obj->size / PAGE_SIZE;
532 if (obj_priv->page_list == NULL)
536 for (i = 0; i < page_count; i++)
537 if (obj_priv->page_list[i] != NULL) {
539 set_page_dirty(obj_priv->page_list[i]);
540 mark_page_accessed(obj_priv->page_list[i]);
541 page_cache_release(obj_priv->page_list[i]);
545 drm_free(obj_priv->page_list,
546 page_count * sizeof(struct page *),
548 obj_priv->page_list = NULL;
552 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
554 struct drm_device *dev = obj->dev;
555 drm_i915_private_t *dev_priv = dev->dev_private;
556 struct drm_i915_gem_object *obj_priv = obj->driver_private;
558 /* Add a reference if we're newly entering the active list. */
559 if (!obj_priv->active) {
560 drm_gem_object_reference(obj);
561 obj_priv->active = 1;
563 /* Move from whatever list we were on to the tail of execution. */
564 list_move_tail(&obj_priv->list,
565 &dev_priv->mm.active_list);
566 obj_priv->last_rendering_seqno = seqno;
570 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
572 struct drm_device *dev = obj->dev;
573 drm_i915_private_t *dev_priv = dev->dev_private;
574 struct drm_i915_gem_object *obj_priv = obj->driver_private;
576 BUG_ON(!obj_priv->active);
577 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
578 obj_priv->last_rendering_seqno = 0;
582 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
584 struct drm_device *dev = obj->dev;
585 drm_i915_private_t *dev_priv = dev->dev_private;
586 struct drm_i915_gem_object *obj_priv = obj->driver_private;
588 i915_verify_inactive(dev, __FILE__, __LINE__);
589 if (obj_priv->pin_count != 0)
590 list_del_init(&obj_priv->list);
592 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
594 obj_priv->last_rendering_seqno = 0;
595 if (obj_priv->active) {
596 obj_priv->active = 0;
597 drm_gem_object_unreference(obj);
599 i915_verify_inactive(dev, __FILE__, __LINE__);
603 * Creates a new sequence number, emitting a write of it to the status page
604 * plus an interrupt, which will trigger i915_user_interrupt_handler.
606 * Must be called with struct_lock held.
608 * Returned sequence numbers are nonzero on success.
611 i915_add_request(struct drm_device *dev, uint32_t flush_domains)
613 drm_i915_private_t *dev_priv = dev->dev_private;
614 struct drm_i915_gem_request *request;
619 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
623 /* Grab the seqno we're going to make this request be, and bump the
624 * next (skipping 0 so it can be the reserved no-seqno value).
626 seqno = dev_priv->mm.next_gem_seqno;
627 dev_priv->mm.next_gem_seqno++;
628 if (dev_priv->mm.next_gem_seqno == 0)
629 dev_priv->mm.next_gem_seqno++;
632 OUT_RING(MI_STORE_DWORD_INDEX);
633 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
636 OUT_RING(MI_USER_INTERRUPT);
639 DRM_DEBUG("%d\n", seqno);
641 request->seqno = seqno;
642 request->emitted_jiffies = jiffies;
643 was_empty = list_empty(&dev_priv->mm.request_list);
644 list_add_tail(&request->list, &dev_priv->mm.request_list);
646 /* Associate any objects on the flushing list matching the write
647 * domain we're flushing with our flush.
649 if (flush_domains != 0) {
650 struct drm_i915_gem_object *obj_priv, *next;
652 list_for_each_entry_safe(obj_priv, next,
653 &dev_priv->mm.flushing_list, list) {
654 struct drm_gem_object *obj = obj_priv->obj;
656 if ((obj->write_domain & flush_domains) ==
658 obj->write_domain = 0;
659 i915_gem_object_move_to_active(obj, seqno);
665 if (was_empty && !dev_priv->mm.suspended)
666 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
671 * Command execution barrier
673 * Ensures that all commands in the ring are finished
674 * before signalling the CPU
677 i915_retire_commands(struct drm_device *dev)
679 drm_i915_private_t *dev_priv = dev->dev_private;
680 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
681 uint32_t flush_domains = 0;
684 /* The sampler always gets flushed on i965 (sigh) */
686 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
689 OUT_RING(0); /* noop */
691 return flush_domains;
695 * Moves buffers associated only with the given active seqno from the active
696 * to inactive list, potentially freeing them.
699 i915_gem_retire_request(struct drm_device *dev,
700 struct drm_i915_gem_request *request)
702 drm_i915_private_t *dev_priv = dev->dev_private;
704 /* Move any buffers on the active list that are no longer referenced
705 * by the ringbuffer to the flushing/inactive lists as appropriate.
707 while (!list_empty(&dev_priv->mm.active_list)) {
708 struct drm_gem_object *obj;
709 struct drm_i915_gem_object *obj_priv;
711 obj_priv = list_first_entry(&dev_priv->mm.active_list,
712 struct drm_i915_gem_object,
716 /* If the seqno being retired doesn't match the oldest in the
717 * list, then the oldest in the list must still be newer than
720 if (obj_priv->last_rendering_seqno != request->seqno)
723 DRM_INFO("%s: retire %d moves to inactive list %p\n",
724 __func__, request->seqno, obj);
727 if (obj->write_domain != 0)
728 i915_gem_object_move_to_flushing(obj);
730 i915_gem_object_move_to_inactive(obj);
735 * Returns true if seq1 is later than seq2.
738 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
740 return (int32_t)(seq1 - seq2) >= 0;
744 i915_get_gem_seqno(struct drm_device *dev)
746 drm_i915_private_t *dev_priv = dev->dev_private;
748 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
752 * This function clears the request list as sequence numbers are passed.
755 i915_gem_retire_requests(struct drm_device *dev)
757 drm_i915_private_t *dev_priv = dev->dev_private;
760 seqno = i915_get_gem_seqno(dev);
762 while (!list_empty(&dev_priv->mm.request_list)) {
763 struct drm_i915_gem_request *request;
764 uint32_t retiring_seqno;
766 request = list_first_entry(&dev_priv->mm.request_list,
767 struct drm_i915_gem_request,
769 retiring_seqno = request->seqno;
771 if (i915_seqno_passed(seqno, retiring_seqno) ||
772 dev_priv->mm.wedged) {
773 i915_gem_retire_request(dev, request);
775 list_del(&request->list);
776 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
783 i915_gem_retire_work_handler(struct work_struct *work)
785 drm_i915_private_t *dev_priv;
786 struct drm_device *dev;
788 dev_priv = container_of(work, drm_i915_private_t,
789 mm.retire_work.work);
792 mutex_lock(&dev->struct_mutex);
793 i915_gem_retire_requests(dev);
794 if (!dev_priv->mm.suspended &&
795 !list_empty(&dev_priv->mm.request_list))
796 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
797 mutex_unlock(&dev->struct_mutex);
801 * Waits for a sequence number to be signaled, and cleans up the
802 * request and object lists appropriately for that event.
805 i915_wait_request(struct drm_device *dev, uint32_t seqno)
807 drm_i915_private_t *dev_priv = dev->dev_private;
812 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
813 dev_priv->mm.waiting_gem_seqno = seqno;
814 i915_user_irq_get(dev);
815 ret = wait_event_interruptible(dev_priv->irq_queue,
816 i915_seqno_passed(i915_get_gem_seqno(dev),
818 dev_priv->mm.wedged);
819 i915_user_irq_put(dev);
820 dev_priv->mm.waiting_gem_seqno = 0;
822 if (dev_priv->mm.wedged)
825 if (ret && ret != -ERESTARTSYS)
826 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
827 __func__, ret, seqno, i915_get_gem_seqno(dev));
829 /* Directly dispatch request retiring. While we have the work queue
830 * to handle this, the waiter on a request often wants an associated
831 * buffer to have made it to the inactive list, and we would need
832 * a separate wait queue to handle that.
835 i915_gem_retire_requests(dev);
841 i915_gem_flush(struct drm_device *dev,
842 uint32_t invalidate_domains,
843 uint32_t flush_domains)
845 drm_i915_private_t *dev_priv = dev->dev_private;
850 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
851 invalidate_domains, flush_domains);
854 if (flush_domains & I915_GEM_DOMAIN_CPU)
855 drm_agp_chipset_flush(dev);
857 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
858 I915_GEM_DOMAIN_GTT)) {
862 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
863 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
864 * also flushed at 2d versus 3d pipeline switches.
868 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
869 * MI_READ_FLUSH is set, and is always flushed on 965.
871 * I915_GEM_DOMAIN_COMMAND may not exist?
873 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
874 * invalidated when MI_EXE_FLUSH is set.
876 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
877 * invalidated with every MI_FLUSH.
881 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
882 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
883 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
884 * are flushed at any MI_FLUSH.
887 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
888 if ((invalidate_domains|flush_domains) &
889 I915_GEM_DOMAIN_RENDER)
890 cmd &= ~MI_NO_WRITE_FLUSH;
891 if (!IS_I965G(dev)) {
893 * On the 965, the sampler cache always gets flushed
894 * and this bit is reserved.
896 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
897 cmd |= MI_READ_FLUSH;
899 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
903 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
907 OUT_RING(0); /* noop */
913 * Ensures that all rendering to the object has completed and the object is
914 * safe to unbind from the GTT or access from the CPU.
917 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
919 struct drm_device *dev = obj->dev;
920 struct drm_i915_gem_object *obj_priv = obj->driver_private;
923 /* This function only exists to support waiting for existing rendering,
924 * not for emitting required flushes.
926 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
928 /* If there is rendering queued on the buffer being evicted, wait for
931 if (obj_priv->active) {
933 DRM_INFO("%s: object %p wait for seqno %08x\n",
934 __func__, obj, obj_priv->last_rendering_seqno);
936 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
945 * Unbinds an object from the GTT aperture.
948 i915_gem_object_unbind(struct drm_gem_object *obj)
950 struct drm_device *dev = obj->dev;
951 struct drm_i915_gem_object *obj_priv = obj->driver_private;
955 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
956 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
958 if (obj_priv->gtt_space == NULL)
961 if (obj_priv->pin_count != 0) {
962 DRM_ERROR("Attempting to unbind pinned buffer\n");
966 /* Move the object to the CPU domain to ensure that
967 * any possible CPU writes while it's not in the GTT
968 * are flushed when we go to remap it. This will
969 * also ensure that all pending GPU writes are finished
972 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
974 if (ret != -ERESTARTSYS)
975 DRM_ERROR("set_domain failed: %d\n", ret);
979 if (obj_priv->agp_mem != NULL) {
980 drm_unbind_agp(obj_priv->agp_mem);
981 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
982 obj_priv->agp_mem = NULL;
985 BUG_ON(obj_priv->active);
987 i915_gem_object_free_page_list(obj);
989 if (obj_priv->gtt_space) {
990 atomic_dec(&dev->gtt_count);
991 atomic_sub(obj->size, &dev->gtt_memory);
993 drm_mm_put_block(obj_priv->gtt_space);
994 obj_priv->gtt_space = NULL;
997 /* Remove ourselves from the LRU list if present. */
998 if (!list_empty(&obj_priv->list))
999 list_del_init(&obj_priv->list);
1005 i915_gem_evict_something(struct drm_device *dev)
1007 drm_i915_private_t *dev_priv = dev->dev_private;
1008 struct drm_gem_object *obj;
1009 struct drm_i915_gem_object *obj_priv;
1013 /* If there's an inactive buffer available now, grab it
1016 if (!list_empty(&dev_priv->mm.inactive_list)) {
1017 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
1018 struct drm_i915_gem_object,
1020 obj = obj_priv->obj;
1021 BUG_ON(obj_priv->pin_count != 0);
1023 DRM_INFO("%s: evicting %p\n", __func__, obj);
1025 BUG_ON(obj_priv->active);
1027 /* Wait on the rendering and unbind the buffer. */
1028 ret = i915_gem_object_unbind(obj);
1032 /* If we didn't get anything, but the ring is still processing
1033 * things, wait for one of those things to finish and hopefully
1034 * leave us a buffer to evict.
1036 if (!list_empty(&dev_priv->mm.request_list)) {
1037 struct drm_i915_gem_request *request;
1039 request = list_first_entry(&dev_priv->mm.request_list,
1040 struct drm_i915_gem_request,
1043 ret = i915_wait_request(dev, request->seqno);
1047 /* if waiting caused an object to become inactive,
1048 * then loop around and wait for it. Otherwise, we
1049 * assume that waiting freed and unbound something,
1050 * so there should now be some space in the GTT
1052 if (!list_empty(&dev_priv->mm.inactive_list))
1057 /* If we didn't have anything on the request list but there
1058 * are buffers awaiting a flush, emit one and try again.
1059 * When we wait on it, those buffers waiting for that flush
1060 * will get moved to inactive.
1062 if (!list_empty(&dev_priv->mm.flushing_list)) {
1063 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1064 struct drm_i915_gem_object,
1066 obj = obj_priv->obj;
1071 i915_add_request(dev, obj->write_domain);
1077 DRM_ERROR("inactive empty %d request empty %d "
1078 "flushing empty %d\n",
1079 list_empty(&dev_priv->mm.inactive_list),
1080 list_empty(&dev_priv->mm.request_list),
1081 list_empty(&dev_priv->mm.flushing_list));
1082 /* If we didn't do any of the above, there's nothing to be done
1083 * and we just can't fit it in.
1091 i915_gem_evict_everything(struct drm_device *dev)
1096 ret = i915_gem_evict_something(dev);
1104 i915_gem_object_get_page_list(struct drm_gem_object *obj)
1106 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1108 struct address_space *mapping;
1109 struct inode *inode;
1113 if (obj_priv->page_list)
1116 /* Get the list of pages out of our struct file. They'll be pinned
1117 * at this point until we release them.
1119 page_count = obj->size / PAGE_SIZE;
1120 BUG_ON(obj_priv->page_list != NULL);
1121 obj_priv->page_list = drm_calloc(page_count, sizeof(struct page *),
1123 if (obj_priv->page_list == NULL) {
1124 DRM_ERROR("Faled to allocate page list\n");
1128 inode = obj->filp->f_path.dentry->d_inode;
1129 mapping = inode->i_mapping;
1130 for (i = 0; i < page_count; i++) {
1131 page = read_mapping_page(mapping, i, NULL);
1133 ret = PTR_ERR(page);
1134 DRM_ERROR("read_mapping_page failed: %d\n", ret);
1135 i915_gem_object_free_page_list(obj);
1138 obj_priv->page_list[i] = page;
1144 * Finds free space in the GTT aperture and binds the object there.
1147 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1149 struct drm_device *dev = obj->dev;
1150 drm_i915_private_t *dev_priv = dev->dev_private;
1151 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1152 struct drm_mm_node *free_space;
1153 int page_count, ret;
1156 alignment = PAGE_SIZE;
1157 if (alignment & (PAGE_SIZE - 1)) {
1158 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1163 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1164 obj->size, alignment, 0);
1165 if (free_space != NULL) {
1166 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
1168 if (obj_priv->gtt_space != NULL) {
1169 obj_priv->gtt_space->private = obj;
1170 obj_priv->gtt_offset = obj_priv->gtt_space->start;
1173 if (obj_priv->gtt_space == NULL) {
1174 /* If the gtt is empty and we're still having trouble
1175 * fitting our object in, we're out of memory.
1178 DRM_INFO("%s: GTT full, evicting something\n", __func__);
1180 if (list_empty(&dev_priv->mm.inactive_list) &&
1181 list_empty(&dev_priv->mm.flushing_list) &&
1182 list_empty(&dev_priv->mm.active_list)) {
1183 DRM_ERROR("GTT full, but LRU list empty\n");
1187 ret = i915_gem_evict_something(dev);
1189 if (ret != -ERESTARTSYS)
1190 DRM_ERROR("Failed to evict a buffer %d\n", ret);
1197 DRM_INFO("Binding object of size %d at 0x%08x\n",
1198 obj->size, obj_priv->gtt_offset);
1200 ret = i915_gem_object_get_page_list(obj);
1202 drm_mm_put_block(obj_priv->gtt_space);
1203 obj_priv->gtt_space = NULL;
1207 page_count = obj->size / PAGE_SIZE;
1208 /* Create an AGP memory structure pointing at our pages, and bind it
1211 obj_priv->agp_mem = drm_agp_bind_pages(dev,
1212 obj_priv->page_list,
1214 obj_priv->gtt_offset,
1215 obj_priv->agp_type);
1216 if (obj_priv->agp_mem == NULL) {
1217 i915_gem_object_free_page_list(obj);
1218 drm_mm_put_block(obj_priv->gtt_space);
1219 obj_priv->gtt_space = NULL;
1222 atomic_inc(&dev->gtt_count);
1223 atomic_add(obj->size, &dev->gtt_memory);
1225 /* Assert that the object is not currently in any GPU domain. As it
1226 * wasn't in the GTT, there shouldn't be any way it could have been in
1229 BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1230 BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1236 i915_gem_clflush_object(struct drm_gem_object *obj)
1238 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1240 /* If we don't have a page list set up, then we're not pinned
1241 * to GPU, and we can ignore the cache flush because it'll happen
1242 * again at bind time.
1244 if (obj_priv->page_list == NULL)
1247 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1250 /** Flushes any GPU write domain for the object if it's dirty. */
1252 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
1254 struct drm_device *dev = obj->dev;
1257 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
1260 /* Queue the GPU write cache flushing we need. */
1261 i915_gem_flush(dev, 0, obj->write_domain);
1262 seqno = i915_add_request(dev, obj->write_domain);
1263 obj->write_domain = 0;
1264 i915_gem_object_move_to_active(obj, seqno);
1267 /** Flushes the GTT write domain for the object if it's dirty. */
1269 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
1271 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
1274 /* No actual flushing is required for the GTT write domain. Writes
1275 * to it immediately go to main memory as far as we know, so there's
1276 * no chipset flush. It also doesn't land in render cache.
1278 obj->write_domain = 0;
1281 /** Flushes the CPU write domain for the object if it's dirty. */
1283 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
1285 struct drm_device *dev = obj->dev;
1287 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
1290 i915_gem_clflush_object(obj);
1291 drm_agp_chipset_flush(dev);
1292 obj->write_domain = 0;
1296 * Moves a single object to the GTT read, and possibly write domain.
1298 * This function returns when the move is complete, including waiting on
1302 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
1304 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1307 i915_gem_object_flush_gpu_write_domain(obj);
1308 /* Wait on any GPU rendering and flushing to occur. */
1309 ret = i915_gem_object_wait_rendering(obj);
1313 /* If we're writing through the GTT domain, then CPU and GPU caches
1314 * will need to be invalidated at next use.
1317 obj->read_domains &= I915_GEM_DOMAIN_GTT;
1319 i915_gem_object_flush_cpu_write_domain(obj);
1321 /* It should now be out of any other write domains, and we can update
1322 * the domain values for our changes.
1324 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
1325 obj->read_domains |= I915_GEM_DOMAIN_GTT;
1327 obj->write_domain = I915_GEM_DOMAIN_GTT;
1328 obj_priv->dirty = 1;
1335 * Moves a single object to the CPU read, and possibly write domain.
1337 * This function returns when the move is complete, including waiting on
1341 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
1343 struct drm_device *dev = obj->dev;
1346 i915_gem_object_flush_gpu_write_domain(obj);
1347 /* Wait on any GPU rendering and flushing to occur. */
1348 ret = i915_gem_object_wait_rendering(obj);
1352 i915_gem_object_flush_gtt_write_domain(obj);
1354 /* If we have a partially-valid cache of the object in the CPU,
1355 * finish invalidating it and free the per-page flags.
1357 i915_gem_object_set_to_full_cpu_read_domain(obj);
1359 /* Flush the CPU cache if it's still invalid. */
1360 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
1361 i915_gem_clflush_object(obj);
1362 drm_agp_chipset_flush(dev);
1364 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1367 /* It should now be out of any other write domains, and we can update
1368 * the domain values for our changes.
1370 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
1372 /* If we're writing through the CPU, then the GPU read domains will
1373 * need to be invalidated at next use.
1376 obj->read_domains &= I915_GEM_DOMAIN_CPU;
1377 obj->write_domain = I915_GEM_DOMAIN_CPU;
1384 * Set the next domain for the specified object. This
1385 * may not actually perform the necessary flushing/invaliding though,
1386 * as that may want to be batched with other set_domain operations
1388 * This is (we hope) the only really tricky part of gem. The goal
1389 * is fairly simple -- track which caches hold bits of the object
1390 * and make sure they remain coherent. A few concrete examples may
1391 * help to explain how it works. For shorthand, we use the notation
1392 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1393 * a pair of read and write domain masks.
1395 * Case 1: the batch buffer
1401 * 5. Unmapped from GTT
1404 * Let's take these a step at a time
1407 * Pages allocated from the kernel may still have
1408 * cache contents, so we set them to (CPU, CPU) always.
1409 * 2. Written by CPU (using pwrite)
1410 * The pwrite function calls set_domain (CPU, CPU) and
1411 * this function does nothing (as nothing changes)
1413 * This function asserts that the object is not
1414 * currently in any GPU-based read or write domains
1416 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
1417 * As write_domain is zero, this function adds in the
1418 * current read domains (CPU+COMMAND, 0).
1419 * flush_domains is set to CPU.
1420 * invalidate_domains is set to COMMAND
1421 * clflush is run to get data out of the CPU caches
1422 * then i915_dev_set_domain calls i915_gem_flush to
1423 * emit an MI_FLUSH and drm_agp_chipset_flush
1424 * 5. Unmapped from GTT
1425 * i915_gem_object_unbind calls set_domain (CPU, CPU)
1426 * flush_domains and invalidate_domains end up both zero
1427 * so no flushing/invalidating happens
1431 * Case 2: The shared render buffer
1435 * 3. Read/written by GPU
1436 * 4. set_domain to (CPU,CPU)
1437 * 5. Read/written by CPU
1438 * 6. Read/written by GPU
1441 * Same as last example, (CPU, CPU)
1443 * Nothing changes (assertions find that it is not in the GPU)
1444 * 3. Read/written by GPU
1445 * execbuffer calls set_domain (RENDER, RENDER)
1446 * flush_domains gets CPU
1447 * invalidate_domains gets GPU
1449 * MI_FLUSH and drm_agp_chipset_flush
1450 * 4. set_domain (CPU, CPU)
1451 * flush_domains gets GPU
1452 * invalidate_domains gets CPU
1453 * wait_rendering (obj) to make sure all drawing is complete.
1454 * This will include an MI_FLUSH to get the data from GPU
1456 * clflush (obj) to invalidate the CPU cache
1457 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1458 * 5. Read/written by CPU
1459 * cache lines are loaded and dirtied
1460 * 6. Read written by GPU
1461 * Same as last GPU access
1463 * Case 3: The constant buffer
1468 * 4. Updated (written) by CPU again
1477 * flush_domains = CPU
1478 * invalidate_domains = RENDER
1481 * drm_agp_chipset_flush
1482 * 4. Updated (written) by CPU again
1484 * flush_domains = 0 (no previous write domain)
1485 * invalidate_domains = 0 (no new read domains)
1488 * flush_domains = CPU
1489 * invalidate_domains = RENDER
1492 * drm_agp_chipset_flush
1495 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
1496 uint32_t read_domains,
1497 uint32_t write_domain)
1499 struct drm_device *dev = obj->dev;
1500 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1501 uint32_t invalidate_domains = 0;
1502 uint32_t flush_domains = 0;
1504 BUG_ON(read_domains & I915_GEM_DOMAIN_CPU);
1505 BUG_ON(write_domain == I915_GEM_DOMAIN_CPU);
1508 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1510 obj->read_domains, read_domains,
1511 obj->write_domain, write_domain);
1514 * If the object isn't moving to a new write domain,
1515 * let the object stay in multiple read domains
1517 if (write_domain == 0)
1518 read_domains |= obj->read_domains;
1520 obj_priv->dirty = 1;
1523 * Flush the current write domain if
1524 * the new read domains don't match. Invalidate
1525 * any read domains which differ from the old
1528 if (obj->write_domain && obj->write_domain != read_domains) {
1529 flush_domains |= obj->write_domain;
1530 invalidate_domains |= read_domains & ~obj->write_domain;
1533 * Invalidate any read caches which may have
1534 * stale data. That is, any new read domains.
1536 invalidate_domains |= read_domains & ~obj->read_domains;
1537 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1539 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1540 __func__, flush_domains, invalidate_domains);
1542 i915_gem_clflush_object(obj);
1545 if ((write_domain | flush_domains) != 0)
1546 obj->write_domain = write_domain;
1547 obj->read_domains = read_domains;
1549 dev->invalidate_domains |= invalidate_domains;
1550 dev->flush_domains |= flush_domains;
1552 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
1554 obj->read_domains, obj->write_domain,
1555 dev->invalidate_domains, dev->flush_domains);
1560 * Moves the object from a partially CPU read to a full one.
1562 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
1563 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
1566 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
1568 struct drm_device *dev = obj->dev;
1569 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1571 if (!obj_priv->page_cpu_valid)
1574 /* If we're partially in the CPU read domain, finish moving it in.
1576 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
1579 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
1580 if (obj_priv->page_cpu_valid[i])
1582 drm_clflush_pages(obj_priv->page_list + i, 1);
1584 drm_agp_chipset_flush(dev);
1587 /* Free the page_cpu_valid mappings which are now stale, whether
1588 * or not we've got I915_GEM_DOMAIN_CPU.
1590 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1592 obj_priv->page_cpu_valid = NULL;
1596 * Set the CPU read domain on a range of the object.
1598 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
1599 * not entirely valid. The page_cpu_valid member of the object flags which
1600 * pages have been flushed, and will be respected by
1601 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
1602 * of the whole object.
1604 * This function returns when the move is complete, including waiting on
1608 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
1609 uint64_t offset, uint64_t size)
1611 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1614 if (offset == 0 && size == obj->size)
1615 return i915_gem_object_set_to_cpu_domain(obj, 0);
1617 i915_gem_object_flush_gpu_write_domain(obj);
1618 /* Wait on any GPU rendering and flushing to occur. */
1619 ret = i915_gem_object_wait_rendering(obj);
1622 i915_gem_object_flush_gtt_write_domain(obj);
1624 /* If we're already fully in the CPU read domain, we're done. */
1625 if (obj_priv->page_cpu_valid == NULL &&
1626 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
1629 /* Otherwise, create/clear the per-page CPU read domain flag if we're
1630 * newly adding I915_GEM_DOMAIN_CPU
1632 if (obj_priv->page_cpu_valid == NULL) {
1633 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1635 if (obj_priv->page_cpu_valid == NULL)
1637 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
1638 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
1640 /* Flush the cache on any pages that are still invalid from the CPU's
1643 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1645 if (obj_priv->page_cpu_valid[i])
1648 drm_clflush_pages(obj_priv->page_list + i, 1);
1650 obj_priv->page_cpu_valid[i] = 1;
1653 /* It should now be out of any other write domains, and we can update
1654 * the domain values for our changes.
1656 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
1658 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1664 * Pin an object to the GTT and evaluate the relocations landing in it.
1667 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1668 struct drm_file *file_priv,
1669 struct drm_i915_gem_exec_object *entry)
1671 struct drm_device *dev = obj->dev;
1672 drm_i915_private_t *dev_priv = dev->dev_private;
1673 struct drm_i915_gem_relocation_entry reloc;
1674 struct drm_i915_gem_relocation_entry __user *relocs;
1675 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1677 void __iomem *reloc_page;
1679 /* Choose the GTT offset for our buffer and put it there. */
1680 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1684 entry->offset = obj_priv->gtt_offset;
1686 relocs = (struct drm_i915_gem_relocation_entry __user *)
1687 (uintptr_t) entry->relocs_ptr;
1688 /* Apply the relocations, using the GTT aperture to avoid cache
1689 * flushing requirements.
1691 for (i = 0; i < entry->relocation_count; i++) {
1692 struct drm_gem_object *target_obj;
1693 struct drm_i915_gem_object *target_obj_priv;
1694 uint32_t reloc_val, reloc_offset;
1695 uint32_t __iomem *reloc_entry;
1697 ret = copy_from_user(&reloc, relocs + i, sizeof(reloc));
1699 i915_gem_object_unpin(obj);
1703 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
1704 reloc.target_handle);
1705 if (target_obj == NULL) {
1706 i915_gem_object_unpin(obj);
1709 target_obj_priv = target_obj->driver_private;
1711 /* The target buffer should have appeared before us in the
1712 * exec_object list, so it should have a GTT space bound by now.
1714 if (target_obj_priv->gtt_space == NULL) {
1715 DRM_ERROR("No GTT space found for object %d\n",
1716 reloc.target_handle);
1717 drm_gem_object_unreference(target_obj);
1718 i915_gem_object_unpin(obj);
1722 if (reloc.offset > obj->size - 4) {
1723 DRM_ERROR("Relocation beyond object bounds: "
1724 "obj %p target %d offset %d size %d.\n",
1725 obj, reloc.target_handle,
1726 (int) reloc.offset, (int) obj->size);
1727 drm_gem_object_unreference(target_obj);
1728 i915_gem_object_unpin(obj);
1731 if (reloc.offset & 3) {
1732 DRM_ERROR("Relocation not 4-byte aligned: "
1733 "obj %p target %d offset %d.\n",
1734 obj, reloc.target_handle,
1735 (int) reloc.offset);
1736 drm_gem_object_unreference(target_obj);
1737 i915_gem_object_unpin(obj);
1741 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
1742 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
1743 DRM_ERROR("reloc with read/write CPU domains: "
1744 "obj %p target %d offset %d "
1745 "read %08x write %08x",
1746 obj, reloc.target_handle,
1749 reloc.write_domain);
1753 if (reloc.write_domain && target_obj->pending_write_domain &&
1754 reloc.write_domain != target_obj->pending_write_domain) {
1755 DRM_ERROR("Write domain conflict: "
1756 "obj %p target %d offset %d "
1757 "new %08x old %08x\n",
1758 obj, reloc.target_handle,
1761 target_obj->pending_write_domain);
1762 drm_gem_object_unreference(target_obj);
1763 i915_gem_object_unpin(obj);
1768 DRM_INFO("%s: obj %p offset %08x target %d "
1769 "read %08x write %08x gtt %08x "
1770 "presumed %08x delta %08x\n",
1774 (int) reloc.target_handle,
1775 (int) reloc.read_domains,
1776 (int) reloc.write_domain,
1777 (int) target_obj_priv->gtt_offset,
1778 (int) reloc.presumed_offset,
1782 target_obj->pending_read_domains |= reloc.read_domains;
1783 target_obj->pending_write_domain |= reloc.write_domain;
1785 /* If the relocation already has the right value in it, no
1786 * more work needs to be done.
1788 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1789 drm_gem_object_unreference(target_obj);
1793 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1795 drm_gem_object_unreference(target_obj);
1796 i915_gem_object_unpin(obj);
1800 /* Map the page containing the relocation we're going to
1803 reloc_offset = obj_priv->gtt_offset + reloc.offset;
1804 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
1807 reloc_entry = (uint32_t __iomem *)(reloc_page +
1808 (reloc_offset & (PAGE_SIZE - 1)));
1809 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1812 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
1813 obj, (unsigned int) reloc.offset,
1814 readl(reloc_entry), reloc_val);
1816 writel(reloc_val, reloc_entry);
1817 io_mapping_unmap_atomic(reloc_page);
1819 /* Write the updated presumed offset for this entry back out
1822 reloc.presumed_offset = target_obj_priv->gtt_offset;
1823 ret = copy_to_user(relocs + i, &reloc, sizeof(reloc));
1825 drm_gem_object_unreference(target_obj);
1826 i915_gem_object_unpin(obj);
1830 drm_gem_object_unreference(target_obj);
1835 i915_gem_dump_object(obj, 128, __func__, ~0);
1840 /** Dispatch a batchbuffer to the ring
1843 i915_dispatch_gem_execbuffer(struct drm_device *dev,
1844 struct drm_i915_gem_execbuffer *exec,
1845 uint64_t exec_offset)
1847 drm_i915_private_t *dev_priv = dev->dev_private;
1848 struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1849 (uintptr_t) exec->cliprects_ptr;
1850 int nbox = exec->num_cliprects;
1852 uint32_t exec_start, exec_len;
1855 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
1856 exec_len = (uint32_t) exec->batch_len;
1858 if ((exec_start | exec_len) & 0x7) {
1859 DRM_ERROR("alignment\n");
1866 count = nbox ? nbox : 1;
1868 for (i = 0; i < count; i++) {
1870 int ret = i915_emit_box(dev, boxes, i,
1871 exec->DR1, exec->DR4);
1876 if (IS_I830(dev) || IS_845G(dev)) {
1878 OUT_RING(MI_BATCH_BUFFER);
1879 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1880 OUT_RING(exec_start + exec_len - 4);
1885 if (IS_I965G(dev)) {
1886 OUT_RING(MI_BATCH_BUFFER_START |
1888 MI_BATCH_NON_SECURE_I965);
1889 OUT_RING(exec_start);
1891 OUT_RING(MI_BATCH_BUFFER_START |
1893 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1899 /* XXX breadcrumb */
1903 /* Throttle our rendering by waiting until the ring has completed our requests
1904 * emitted over 20 msec ago.
1906 * This should get us reasonable parallelism between CPU and GPU but also
1907 * relatively low latency when blocking on a particular request to finish.
1910 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1912 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1916 mutex_lock(&dev->struct_mutex);
1917 seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1918 i915_file_priv->mm.last_gem_throttle_seqno =
1919 i915_file_priv->mm.last_gem_seqno;
1921 ret = i915_wait_request(dev, seqno);
1922 mutex_unlock(&dev->struct_mutex);
1927 i915_gem_execbuffer(struct drm_device *dev, void *data,
1928 struct drm_file *file_priv)
1930 drm_i915_private_t *dev_priv = dev->dev_private;
1931 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1932 struct drm_i915_gem_execbuffer *args = data;
1933 struct drm_i915_gem_exec_object *exec_list = NULL;
1934 struct drm_gem_object **object_list = NULL;
1935 struct drm_gem_object *batch_obj;
1936 int ret, i, pinned = 0;
1937 uint64_t exec_offset;
1938 uint32_t seqno, flush_domains;
1942 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1943 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1946 if (args->buffer_count < 1) {
1947 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1950 /* Copy in the exec list from userland */
1951 exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
1953 object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
1955 if (exec_list == NULL || object_list == NULL) {
1956 DRM_ERROR("Failed to allocate exec or object list "
1958 args->buffer_count);
1962 ret = copy_from_user(exec_list,
1963 (struct drm_i915_relocation_entry __user *)
1964 (uintptr_t) args->buffers_ptr,
1965 sizeof(*exec_list) * args->buffer_count);
1967 DRM_ERROR("copy %d exec entries failed %d\n",
1968 args->buffer_count, ret);
1972 mutex_lock(&dev->struct_mutex);
1974 i915_verify_inactive(dev, __FILE__, __LINE__);
1976 if (dev_priv->mm.wedged) {
1977 DRM_ERROR("Execbuf while wedged\n");
1978 mutex_unlock(&dev->struct_mutex);
1982 if (dev_priv->mm.suspended) {
1983 DRM_ERROR("Execbuf while VT-switched.\n");
1984 mutex_unlock(&dev->struct_mutex);
1988 /* Look up object handles */
1989 for (i = 0; i < args->buffer_count; i++) {
1990 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1991 exec_list[i].handle);
1992 if (object_list[i] == NULL) {
1993 DRM_ERROR("Invalid object handle %d at index %d\n",
1994 exec_list[i].handle, i);
2000 /* Pin and relocate */
2001 for (pin_tries = 0; ; pin_tries++) {
2003 for (i = 0; i < args->buffer_count; i++) {
2004 object_list[i]->pending_read_domains = 0;
2005 object_list[i]->pending_write_domain = 0;
2006 ret = i915_gem_object_pin_and_relocate(object_list[i],
2017 /* error other than GTT full, or we've already tried again */
2018 if (ret != -ENOMEM || pin_tries >= 1) {
2019 DRM_ERROR("Failed to pin buffers %d\n", ret);
2023 /* unpin all of our buffers */
2024 for (i = 0; i < pinned; i++)
2025 i915_gem_object_unpin(object_list[i]);
2027 /* evict everyone we can from the aperture */
2028 ret = i915_gem_evict_everything(dev);
2033 /* Set the pending read domains for the batch buffer to COMMAND */
2034 batch_obj = object_list[args->buffer_count-1];
2035 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
2036 batch_obj->pending_write_domain = 0;
2038 i915_verify_inactive(dev, __FILE__, __LINE__);
2040 /* Zero the global flush/invalidate flags. These
2041 * will be modified as new domains are computed
2044 dev->invalidate_domains = 0;
2045 dev->flush_domains = 0;
2047 for (i = 0; i < args->buffer_count; i++) {
2048 struct drm_gem_object *obj = object_list[i];
2050 /* Compute new gpu domains and update invalidate/flush */
2051 i915_gem_object_set_to_gpu_domain(obj,
2052 obj->pending_read_domains,
2053 obj->pending_write_domain);
2056 i915_verify_inactive(dev, __FILE__, __LINE__);
2058 if (dev->invalidate_domains | dev->flush_domains) {
2060 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
2062 dev->invalidate_domains,
2063 dev->flush_domains);
2066 dev->invalidate_domains,
2067 dev->flush_domains);
2068 if (dev->flush_domains)
2069 (void)i915_add_request(dev, dev->flush_domains);
2072 i915_verify_inactive(dev, __FILE__, __LINE__);
2075 for (i = 0; i < args->buffer_count; i++) {
2076 i915_gem_object_check_coherency(object_list[i],
2077 exec_list[i].handle);
2081 exec_offset = exec_list[args->buffer_count - 1].offset;
2084 i915_gem_dump_object(object_list[args->buffer_count - 1],
2090 /* Exec the batchbuffer */
2091 ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
2093 DRM_ERROR("dispatch failed %d\n", ret);
2098 * Ensure that the commands in the batch buffer are
2099 * finished before the interrupt fires
2101 flush_domains = i915_retire_commands(dev);
2103 i915_verify_inactive(dev, __FILE__, __LINE__);
2106 * Get a seqno representing the execution of the current buffer,
2107 * which we can wait on. We would like to mitigate these interrupts,
2108 * likely by only creating seqnos occasionally (so that we have
2109 * *some* interrupts representing completion of buffers that we can
2110 * wait on when trying to clear up gtt space).
2112 seqno = i915_add_request(dev, flush_domains);
2114 i915_file_priv->mm.last_gem_seqno = seqno;
2115 for (i = 0; i < args->buffer_count; i++) {
2116 struct drm_gem_object *obj = object_list[i];
2118 i915_gem_object_move_to_active(obj, seqno);
2120 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
2124 i915_dump_lru(dev, __func__);
2127 i915_verify_inactive(dev, __FILE__, __LINE__);
2129 /* Copy the new buffer offsets back to the user's exec list. */
2130 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
2131 (uintptr_t) args->buffers_ptr,
2133 sizeof(*exec_list) * args->buffer_count);
2135 DRM_ERROR("failed to copy %d exec entries "
2136 "back to user (%d)\n",
2137 args->buffer_count, ret);
2139 if (object_list != NULL) {
2140 for (i = 0; i < pinned; i++)
2141 i915_gem_object_unpin(object_list[i]);
2143 for (i = 0; i < args->buffer_count; i++)
2144 drm_gem_object_unreference(object_list[i]);
2146 mutex_unlock(&dev->struct_mutex);
2149 drm_free(object_list, sizeof(*object_list) * args->buffer_count,
2151 drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
2158 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
2160 struct drm_device *dev = obj->dev;
2161 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2164 i915_verify_inactive(dev, __FILE__, __LINE__);
2165 if (obj_priv->gtt_space == NULL) {
2166 ret = i915_gem_object_bind_to_gtt(obj, alignment);
2168 DRM_ERROR("Failure to bind: %d", ret);
2172 obj_priv->pin_count++;
2174 /* If the object is not active and not pending a flush,
2175 * remove it from the inactive list
2177 if (obj_priv->pin_count == 1) {
2178 atomic_inc(&dev->pin_count);
2179 atomic_add(obj->size, &dev->pin_memory);
2180 if (!obj_priv->active &&
2181 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2182 I915_GEM_DOMAIN_GTT)) == 0 &&
2183 !list_empty(&obj_priv->list))
2184 list_del_init(&obj_priv->list);
2186 i915_verify_inactive(dev, __FILE__, __LINE__);
2192 i915_gem_object_unpin(struct drm_gem_object *obj)
2194 struct drm_device *dev = obj->dev;
2195 drm_i915_private_t *dev_priv = dev->dev_private;
2196 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2198 i915_verify_inactive(dev, __FILE__, __LINE__);
2199 obj_priv->pin_count--;
2200 BUG_ON(obj_priv->pin_count < 0);
2201 BUG_ON(obj_priv->gtt_space == NULL);
2203 /* If the object is no longer pinned, and is
2204 * neither active nor being flushed, then stick it on
2207 if (obj_priv->pin_count == 0) {
2208 if (!obj_priv->active &&
2209 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2210 I915_GEM_DOMAIN_GTT)) == 0)
2211 list_move_tail(&obj_priv->list,
2212 &dev_priv->mm.inactive_list);
2213 atomic_dec(&dev->pin_count);
2214 atomic_sub(obj->size, &dev->pin_memory);
2216 i915_verify_inactive(dev, __FILE__, __LINE__);
2220 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2221 struct drm_file *file_priv)
2223 struct drm_i915_gem_pin *args = data;
2224 struct drm_gem_object *obj;
2225 struct drm_i915_gem_object *obj_priv;
2228 mutex_lock(&dev->struct_mutex);
2230 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2232 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2234 mutex_unlock(&dev->struct_mutex);
2237 obj_priv = obj->driver_private;
2239 ret = i915_gem_object_pin(obj, args->alignment);
2241 drm_gem_object_unreference(obj);
2242 mutex_unlock(&dev->struct_mutex);
2246 /* XXX - flush the CPU caches for pinned objects
2247 * as the X server doesn't manage domains yet
2249 i915_gem_object_flush_cpu_write_domain(obj);
2250 args->offset = obj_priv->gtt_offset;
2251 drm_gem_object_unreference(obj);
2252 mutex_unlock(&dev->struct_mutex);
2258 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2259 struct drm_file *file_priv)
2261 struct drm_i915_gem_pin *args = data;
2262 struct drm_gem_object *obj;
2264 mutex_lock(&dev->struct_mutex);
2266 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2268 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2270 mutex_unlock(&dev->struct_mutex);
2274 i915_gem_object_unpin(obj);
2276 drm_gem_object_unreference(obj);
2277 mutex_unlock(&dev->struct_mutex);
2282 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2283 struct drm_file *file_priv)
2285 struct drm_i915_gem_busy *args = data;
2286 struct drm_gem_object *obj;
2287 struct drm_i915_gem_object *obj_priv;
2289 mutex_lock(&dev->struct_mutex);
2290 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2292 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2294 mutex_unlock(&dev->struct_mutex);
2298 obj_priv = obj->driver_private;
2299 args->busy = obj_priv->active;
2301 drm_gem_object_unreference(obj);
2302 mutex_unlock(&dev->struct_mutex);
2307 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2308 struct drm_file *file_priv)
2310 return i915_gem_ring_throttle(dev, file_priv);
2313 int i915_gem_init_object(struct drm_gem_object *obj)
2315 struct drm_i915_gem_object *obj_priv;
2317 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2318 if (obj_priv == NULL)
2322 * We've just allocated pages from the kernel,
2323 * so they've just been written by the CPU with
2324 * zeros. They'll need to be clflushed before we
2325 * use them with the GPU.
2327 obj->write_domain = I915_GEM_DOMAIN_CPU;
2328 obj->read_domains = I915_GEM_DOMAIN_CPU;
2330 obj_priv->agp_type = AGP_USER_MEMORY;
2332 obj->driver_private = obj_priv;
2333 obj_priv->obj = obj;
2334 INIT_LIST_HEAD(&obj_priv->list);
2338 void i915_gem_free_object(struct drm_gem_object *obj)
2340 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2342 while (obj_priv->pin_count > 0)
2343 i915_gem_object_unpin(obj);
2345 i915_gem_object_unbind(obj);
2347 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
2348 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2351 /** Unbinds all objects that are on the given buffer list. */
2353 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
2355 struct drm_gem_object *obj;
2356 struct drm_i915_gem_object *obj_priv;
2359 while (!list_empty(head)) {
2360 obj_priv = list_first_entry(head,
2361 struct drm_i915_gem_object,
2363 obj = obj_priv->obj;
2365 if (obj_priv->pin_count != 0) {
2366 DRM_ERROR("Pinned object in unbind list\n");
2367 mutex_unlock(&dev->struct_mutex);
2371 ret = i915_gem_object_unbind(obj);
2373 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2375 mutex_unlock(&dev->struct_mutex);
2385 i915_gem_idle(struct drm_device *dev)
2387 drm_i915_private_t *dev_priv = dev->dev_private;
2388 uint32_t seqno, cur_seqno, last_seqno;
2391 mutex_lock(&dev->struct_mutex);
2393 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
2394 mutex_unlock(&dev->struct_mutex);
2398 /* Hack! Don't let anybody do execbuf while we don't control the chip.
2399 * We need to replace this with a semaphore, or something.
2401 dev_priv->mm.suspended = 1;
2403 /* Cancel the retire work handler, wait for it to finish if running
2405 mutex_unlock(&dev->struct_mutex);
2406 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2407 mutex_lock(&dev->struct_mutex);
2409 i915_kernel_lost_context(dev);
2411 /* Flush the GPU along with all non-CPU write domains
2413 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2414 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2415 seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2416 I915_GEM_DOMAIN_GTT));
2419 mutex_unlock(&dev->struct_mutex);
2423 dev_priv->mm.waiting_gem_seqno = seqno;
2427 cur_seqno = i915_get_gem_seqno(dev);
2428 if (i915_seqno_passed(cur_seqno, seqno))
2430 if (last_seqno == cur_seqno) {
2431 if (stuck++ > 100) {
2432 DRM_ERROR("hardware wedged\n");
2433 dev_priv->mm.wedged = 1;
2434 DRM_WAKEUP(&dev_priv->irq_queue);
2439 last_seqno = cur_seqno;
2441 dev_priv->mm.waiting_gem_seqno = 0;
2443 i915_gem_retire_requests(dev);
2445 if (!dev_priv->mm.wedged) {
2446 /* Active and flushing should now be empty as we've
2447 * waited for a sequence higher than any pending execbuffer
2449 WARN_ON(!list_empty(&dev_priv->mm.active_list));
2450 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
2451 /* Request should now be empty as we've also waited
2452 * for the last request in the list
2454 WARN_ON(!list_empty(&dev_priv->mm.request_list));
2457 /* Empty the active and flushing lists to inactive. If there's
2458 * anything left at this point, it means that we're wedged and
2459 * nothing good's going to happen by leaving them there. So strip
2460 * the GPU domains and just stuff them onto inactive.
2462 while (!list_empty(&dev_priv->mm.active_list)) {
2463 struct drm_i915_gem_object *obj_priv;
2465 obj_priv = list_first_entry(&dev_priv->mm.active_list,
2466 struct drm_i915_gem_object,
2468 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2469 i915_gem_object_move_to_inactive(obj_priv->obj);
2472 while (!list_empty(&dev_priv->mm.flushing_list)) {
2473 struct drm_i915_gem_object *obj_priv;
2475 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
2476 struct drm_i915_gem_object,
2478 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2479 i915_gem_object_move_to_inactive(obj_priv->obj);
2483 /* Move all inactive buffers out of the GTT. */
2484 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
2485 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
2487 mutex_unlock(&dev->struct_mutex);
2491 i915_gem_cleanup_ringbuffer(dev);
2492 mutex_unlock(&dev->struct_mutex);
2498 i915_gem_init_hws(struct drm_device *dev)
2500 drm_i915_private_t *dev_priv = dev->dev_private;
2501 struct drm_gem_object *obj;
2502 struct drm_i915_gem_object *obj_priv;
2505 /* If we need a physical address for the status page, it's already
2506 * initialized at driver load time.
2508 if (!I915_NEED_GFX_HWS(dev))
2511 obj = drm_gem_object_alloc(dev, 4096);
2513 DRM_ERROR("Failed to allocate status page\n");
2516 obj_priv = obj->driver_private;
2517 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
2519 ret = i915_gem_object_pin(obj, 4096);
2521 drm_gem_object_unreference(obj);
2525 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
2527 dev_priv->hw_status_page = kmap(obj_priv->page_list[0]);
2528 if (dev_priv->hw_status_page == NULL) {
2529 DRM_ERROR("Failed to map status page.\n");
2530 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2531 drm_gem_object_unreference(obj);
2534 dev_priv->hws_obj = obj;
2535 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2536 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
2537 I915_READ(HWS_PGA); /* posting read */
2538 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2544 i915_gem_init_ringbuffer(struct drm_device *dev)
2546 drm_i915_private_t *dev_priv = dev->dev_private;
2547 struct drm_gem_object *obj;
2548 struct drm_i915_gem_object *obj_priv;
2552 ret = i915_gem_init_hws(dev);
2556 obj = drm_gem_object_alloc(dev, 128 * 1024);
2558 DRM_ERROR("Failed to allocate ringbuffer\n");
2561 obj_priv = obj->driver_private;
2563 ret = i915_gem_object_pin(obj, 4096);
2565 drm_gem_object_unreference(obj);
2569 /* Set up the kernel mapping for the ring. */
2570 dev_priv->ring.Size = obj->size;
2571 dev_priv->ring.tail_mask = obj->size - 1;
2573 dev_priv->ring.map.offset = dev->agp->base + obj_priv->gtt_offset;
2574 dev_priv->ring.map.size = obj->size;
2575 dev_priv->ring.map.type = 0;
2576 dev_priv->ring.map.flags = 0;
2577 dev_priv->ring.map.mtrr = 0;
2579 drm_core_ioremap_wc(&dev_priv->ring.map, dev);
2580 if (dev_priv->ring.map.handle == NULL) {
2581 DRM_ERROR("Failed to map ringbuffer.\n");
2582 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2583 drm_gem_object_unreference(obj);
2586 dev_priv->ring.ring_obj = obj;
2587 dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
2589 /* Stop the ring if it's running. */
2590 I915_WRITE(PRB0_CTL, 0);
2591 I915_WRITE(PRB0_TAIL, 0);
2592 I915_WRITE(PRB0_HEAD, 0);
2594 /* Initialize the ring. */
2595 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
2596 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2598 /* G45 ring initialization fails to reset head to zero */
2600 DRM_ERROR("Ring head not reset to zero "
2601 "ctl %08x head %08x tail %08x start %08x\n",
2602 I915_READ(PRB0_CTL),
2603 I915_READ(PRB0_HEAD),
2604 I915_READ(PRB0_TAIL),
2605 I915_READ(PRB0_START));
2606 I915_WRITE(PRB0_HEAD, 0);
2608 DRM_ERROR("Ring head forced to zero "
2609 "ctl %08x head %08x tail %08x start %08x\n",
2610 I915_READ(PRB0_CTL),
2611 I915_READ(PRB0_HEAD),
2612 I915_READ(PRB0_TAIL),
2613 I915_READ(PRB0_START));
2616 I915_WRITE(PRB0_CTL,
2617 ((obj->size - 4096) & RING_NR_PAGES) |
2621 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2623 /* If the head is still not zero, the ring is dead */
2625 DRM_ERROR("Ring initialization failed "
2626 "ctl %08x head %08x tail %08x start %08x\n",
2627 I915_READ(PRB0_CTL),
2628 I915_READ(PRB0_HEAD),
2629 I915_READ(PRB0_TAIL),
2630 I915_READ(PRB0_START));
2634 /* Update our cache of the ring state */
2635 i915_kernel_lost_context(dev);
2641 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2643 drm_i915_private_t *dev_priv = dev->dev_private;
2645 if (dev_priv->ring.ring_obj == NULL)
2648 drm_core_ioremapfree(&dev_priv->ring.map, dev);
2650 i915_gem_object_unpin(dev_priv->ring.ring_obj);
2651 drm_gem_object_unreference(dev_priv->ring.ring_obj);
2652 dev_priv->ring.ring_obj = NULL;
2653 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2655 if (dev_priv->hws_obj != NULL) {
2656 struct drm_gem_object *obj = dev_priv->hws_obj;
2657 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2659 kunmap(obj_priv->page_list[0]);
2660 i915_gem_object_unpin(obj);
2661 drm_gem_object_unreference(obj);
2662 dev_priv->hws_obj = NULL;
2663 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2664 dev_priv->hw_status_page = NULL;
2666 /* Write high address into HWS_PGA when disabling. */
2667 I915_WRITE(HWS_PGA, 0x1ffff000);
2672 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2673 struct drm_file *file_priv)
2675 drm_i915_private_t *dev_priv = dev->dev_private;
2678 if (dev_priv->mm.wedged) {
2679 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2680 dev_priv->mm.wedged = 0;
2683 ret = i915_gem_init_ringbuffer(dev);
2687 dev_priv->mm.gtt_mapping = io_mapping_create_wc(dev->agp->base,
2688 dev->agp->agp_info.aper_size
2691 mutex_lock(&dev->struct_mutex);
2692 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2693 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2694 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2695 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2696 dev_priv->mm.suspended = 0;
2697 mutex_unlock(&dev->struct_mutex);
2699 drm_irq_install(dev);
2705 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2706 struct drm_file *file_priv)
2708 drm_i915_private_t *dev_priv = dev->dev_private;
2711 ret = i915_gem_idle(dev);
2712 drm_irq_uninstall(dev);
2714 io_mapping_free(dev_priv->mm.gtt_mapping);
2719 i915_gem_lastclose(struct drm_device *dev)
2723 ret = i915_gem_idle(dev);
2725 DRM_ERROR("failed to idle hardware: %d\n", ret);
2729 i915_gem_load(struct drm_device *dev)
2731 drm_i915_private_t *dev_priv = dev->dev_private;
2733 INIT_LIST_HEAD(&dev_priv->mm.active_list);
2734 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2735 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2736 INIT_LIST_HEAD(&dev_priv->mm.request_list);
2737 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2738 i915_gem_retire_work_handler);
2739 dev_priv->mm.next_gem_seqno = 1;
2741 i915_gem_detect_bit_6_swizzle(dev);