- if (pwake)
- ep_poll_safewake(&psw, &ep->poll_wait);
-
- return 0;
-}
-
-
-/*
- * This function unregister poll callbacks from the associated file descriptor.
- * Since this must be called without holding "ep->lock" the atomic exchange trick
- * will protect us from multiple unregister.
- */
-static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
-{
- int nwait;
- struct list_head *lsthead = &epi->pwqlist;
- struct eppoll_entry *pwq;
-
- /* This is called without locks, so we need the atomic exchange */
- nwait = xchg(&epi->nwait, 0);
-
- if (nwait) {
- while (!list_empty(lsthead)) {
- pwq = list_entry(lsthead->next, struct eppoll_entry, llink);
-
- ep_list_del(&pwq->llink);
- remove_wait_queue(pwq->whead, &pwq->wait);
- kmem_cache_free(pwq_cache, pwq);
- }
- }
-}
-
-
-/*
- * Unlink the "struct epitem" from all places it might have been hooked up.
- * This function must be called with write IRQ lock on "ep->lock".
- */
-static int ep_unlink(struct eventpoll *ep, struct epitem *epi)
-{
- int error;
-
- /*
- * It can happen that this one is called for an item already unlinked.
- * The check protect us from doing a double unlink ( crash ).
- */
- error = -ENOENT;
- if (!ep_rb_linked(&epi->rbn))
- goto eexit_1;
-
- /*
- * Clear the event mask for the unlinked item. This will avoid item
- * notifications to be sent after the unlink operation from inside
- * the kernel->userspace event transfer loop.
- */
- epi->event.events = 0;
-
- /*
- * At this point is safe to do the job, unlink the item from our rb-tree.
- * This operation togheter with the above check closes the door to
- * double unlinks.
- */
- ep_rb_erase(&epi->rbn, &ep->rbr);
-
- /*
- * If the item we are going to remove is inside the ready file descriptors
- * we want to remove it from this list to avoid stale events.
- */
- if (ep_is_linked(&epi->rdllink))
- ep_list_del(&epi->rdllink);
-
- error = 0;
-eexit_1:
-
- DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_unlink(%p, %p) = %d\n",
- current, ep, epi->ffd.file, error));
-
- return error;
-}
-
-
-/*
- * Removes a "struct epitem" from the eventpoll hash and deallocates
- * all the associated resources.
- */
-static int ep_remove(struct eventpoll *ep, struct epitem *epi)
-{
- int error;
- unsigned long flags;
- struct file *file = epi->ffd.file;
-
- /*
- * Removes poll wait queue hooks. We _have_ to do this without holding
- * the "ep->lock" otherwise a deadlock might occur. This because of the
- * sequence of the lock acquisition. Here we do "ep->lock" then the wait
- * queue head lock when unregistering the wait queue. The wakeup callback
- * will run by holding the wait queue head lock and will call our callback
- * that will try to get "ep->lock".
- */
- ep_unregister_pollwait(ep, epi);
-
- /* Remove the current item from the list of epoll hooks */
- spin_lock(&file->f_ep_lock);
- if (ep_is_linked(&epi->fllink))
- ep_list_del(&epi->fllink);
- spin_unlock(&file->f_ep_lock);
-
- /* We need to acquire the write IRQ lock before calling ep_unlink() */
- write_lock_irqsave(&ep->lock, flags);
-
- /* Really unlink the item from the hash */
- error = ep_unlink(ep, epi);
-
- write_unlock_irqrestore(&ep->lock, flags);
-
- if (error)
- goto eexit_1;
-
- /* At this point it is safe to free the eventpoll item */
- ep_release_epitem(epi);
-
- error = 0;
-eexit_1:
- DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_remove(%p, %p) = %d\n",
- current, ep, file, error));
-
- return error;
-}
-
-
-/*
- * This is the callback that is passed to the wait queue wakeup
- * machanism. It is called by the stored file descriptors when they
- * have events to report.
- */
-static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *key)
-{
- int pwake = 0;
- unsigned long flags;
- struct epitem *epi = ep_item_from_wait(wait);
- struct eventpoll *ep = epi->ep;
-
- DNPRINTK(3, (KERN_INFO "[%p] eventpoll: poll_callback(%p) epi=%p ep=%p\n",
- current, epi->ffd.file, epi, ep));
-
- write_lock_irqsave(&ep->lock, flags);
-
- /*
- * If the event mask does not contain any poll(2) event, we consider the
- * descriptor to be disabled. This condition is likely the effect of the
- * EPOLLONESHOT bit that disables the descriptor when an event is received,
- * until the next EPOLL_CTL_MOD will be issued.
- */
- if (!(epi->event.events & ~EP_PRIVATE_BITS))
- goto is_disabled;
-
- /* If this file is already in the ready list we exit soon */
- if (ep_is_linked(&epi->rdllink))
- goto is_linked;
-
- list_add_tail(&epi->rdllink, &ep->rdllist);
-
-is_linked:
- /*
- * Wake up ( if active ) both the eventpoll wait list and the ->poll()
- * wait list.
- */
- if (waitqueue_active(&ep->wq))
- __wake_up_locked(&ep->wq, TASK_UNINTERRUPTIBLE |
- TASK_INTERRUPTIBLE);
- if (waitqueue_active(&ep->poll_wait))
- pwake++;
-
-is_disabled:
- write_unlock_irqrestore(&ep->lock, flags);
-
- /* We have to call this outside the lock */
- if (pwake)
- ep_poll_safewake(&psw, &ep->poll_wait);
-
- return 1;
-}
-
-
-static int ep_eventpoll_close(struct inode *inode, struct file *file)
-{
- struct eventpoll *ep = file->private_data;
-
- if (ep) {
- ep_free(ep);
- kfree(ep);
- }
-
- DNPRINTK(3, (KERN_INFO "[%p] eventpoll: close() ep=%p\n", current, ep));
- return 0;
-}
-
-
-static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait)
-{
- unsigned int pollflags = 0;
- unsigned long flags;
- struct eventpoll *ep = file->private_data;
-
- /* Insert inside our poll wait queue */
- poll_wait(file, &ep->poll_wait, wait);
-
- /* Check our condition */
- read_lock_irqsave(&ep->lock, flags);
- if (!list_empty(&ep->rdllist))
- pollflags = POLLIN | POLLRDNORM;
- read_unlock_irqrestore(&ep->lock, flags);
-
- return pollflags;
-}
-
-
-/*
- * Since we have to release the lock during the __copy_to_user() operation and
- * during the f_op->poll() call, we try to collect the maximum number of items
- * by reducing the irqlock/irqunlock switching rate.
- */
-static int ep_collect_ready_items(struct eventpoll *ep, struct list_head *txlist, int maxevents)
-{
- int nepi;
- unsigned long flags;
- struct list_head *lsthead = &ep->rdllist, *lnk;
- struct epitem *epi;
-
- write_lock_irqsave(&ep->lock, flags);
-
- for (nepi = 0, lnk = lsthead->next; lnk != lsthead && nepi < maxevents;) {
- epi = list_entry(lnk, struct epitem, rdllink);
-
- lnk = lnk->next;
-
- /* If this file is already in the ready list we exit soon */
- if (!ep_is_linked(&epi->txlink)) {
- /*
- * This is initialized in this way so that the default
- * behaviour of the reinjecting code will be to push back
- * the item inside the ready list.
- */
- epi->revents = epi->event.events;
-
- /* Link the ready item into the transfer list */
- list_add(&epi->txlink, txlist);
- nepi++;
-
- /*
- * Unlink the item from the ready list.
- */
- ep_list_del(&epi->rdllink);
- }
- }
-
- write_unlock_irqrestore(&ep->lock, flags);
-
- return nepi;
-}