SAFE public projects git trees. - safe/jmp/linux-2.6/blob - fs/splice.c

   1 /*
   2  * "splice": joining two ropes together by interweaving their strands.
   3  *
   4  * This is the "extended pipe" functionality, where a pipe is used as
   5  * an arbitrary in-memory buffer. Think of a pipe as a small kernel
   6  * buffer that you can use to transfer data from one end to the other.
   7  *
   8  * The traditional unix read/write is extended with a "splice()" operation
   9  * that transfers data buffers to or from a pipe buffer.
  10  *
  11  * Named by Larry McVoy, original implementation from Linus, extended by
  12  * Jens to support splicing to files, network, direct splicing, etc and
  13  * fixing lots of bugs.
  14  *
  15  * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
  16  * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
  17  * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  18  *
  19  */
  20 #include <linux/fs.h>
  21 #include <linux/file.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/pipe_fs_i.h>
  24 #include <linux/mm_inline.h>
  25 #include <linux/swap.h>
  26 #include <linux/writeback.h>
  27 #include <linux/buffer_head.h>
  28 #include <linux/module.h>
  29 #include <linux/syscalls.h>
  30 #include <linux/uio.h>
  31
  32 struct partial_page {
  33         unsigned int offset;
  34         unsigned int len;
  35 };
  36
  37 /*
  38  * Passed to splice_to_pipe
  39  */
  40 struct splice_pipe_desc {
  41         struct page **pages;            /* page map */
  42         struct partial_page *partial;   /* pages[] may not be contig */
  43         int nr_pages;                   /* number of pages in map */
  44         unsigned int flags;             /* splice flags */
  45         struct pipe_buf_operations *ops;/* ops associated with output pipe */
  46 };
  47
  48 /*
  49  * Attempt to steal a page from a pipe buffer. This should perhaps go into
  50  * a vm helper function, it's already simplified quite a bit by the
  51  * addition of remove_mapping(). If success is returned, the caller may
  52  * attempt to reuse this page for another destination.
  53  */
  54 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
  55                                      struct pipe_buffer *buf)
  56 {
  57         struct page *page = buf->page;
  58         struct address_space *mapping = page_mapping(page);
  59
  60         lock_page(page);
  61
  62         WARN_ON(!PageUptodate(page));
  63
  64         /*
  65          * At least for ext2 with nobh option, we need to wait on writeback
  66          * completing on this page, since we'll remove it from the pagecache.
  67          * Otherwise truncate wont wait on the page, allowing the disk
  68          * blocks to be reused by someone else before we actually wrote our
  69          * data to them. fs corruption ensues.
  70          */
  71         wait_on_page_writeback(page);
  72
  73         if (PagePrivate(page))
  74                 try_to_release_page(page, mapping_gfp_mask(mapping));
  75
  76         if (!remove_mapping(mapping, page)) {
  77                 unlock_page(page);
  78                 return 1;
  79         }
  80
  81         buf->flags |= PIPE_BUF_FLAG_LRU;
  82         return 0;
  83 }
  84
  85 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
  86                                         struct pipe_buffer *buf)
  87 {
  88         page_cache_release(buf->page);
  89         buf->flags &= ~PIPE_BUF_FLAG_LRU;
  90 }
  91
  92 static int page_cache_pipe_buf_pin(struct pipe_inode_info *pipe,
  93                                    struct pipe_buffer *buf)
  94 {
  95         struct page *page = buf->page;
  96         int err;
  97
  98         if (!PageUptodate(page)) {
  99                 lock_page(page);
 100
 101                 /*
 102                  * Page got truncated/unhashed. This will cause a 0-byte
 103                  * splice, if this is the first page.
 104                  */
 105                 if (!page->mapping) {
 106                         err = -ENODATA;
 107                         goto error;
 108                 }
 109
 110                 /*
 111                  * Uh oh, read-error from disk.
 112                  */
 113                 if (!PageUptodate(page)) {
 114                         err = -EIO;
 115                         goto error;
 116                 }
 117
 118                 /*
 119                  * Page is ok afterall, we are done.
 120                  */
 121                 unlock_page(page);
 122         }
 123
 124         return 0;
 125 error:
 126         unlock_page(page);
 127         return err;
 128 }
 129
 130 static struct pipe_buf_operations page_cache_pipe_buf_ops = {
 131         .can_merge = 0,
 132         .map = generic_pipe_buf_map,
 133         .unmap = generic_pipe_buf_unmap,
 134         .pin = page_cache_pipe_buf_pin,
 135         .release = page_cache_pipe_buf_release,
 136         .steal = page_cache_pipe_buf_steal,
 137         .get = generic_pipe_buf_get,
 138 };
 139
 140 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
 141                                     struct pipe_buffer *buf)
 142 {
 143         if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
 144                 return 1;
 145
 146         buf->flags |= PIPE_BUF_FLAG_LRU;
 147         return generic_pipe_buf_steal(pipe, buf);
 148 }
 149
 150 static struct pipe_buf_operations user_page_pipe_buf_ops = {
 151         .can_merge = 0,
 152         .map = generic_pipe_buf_map,
 153         .unmap = generic_pipe_buf_unmap,
 154         .pin = generic_pipe_buf_pin,
 155         .release = page_cache_pipe_buf_release,
 156         .steal = user_page_pipe_buf_steal,
 157         .get = generic_pipe_buf_get,
 158 };
 159
 160 /*
 161  * Pipe output worker. This sets up our pipe format with the page cache
 162  * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
 163  */
 164 static ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
 165                               struct splice_pipe_desc *spd)
 166 {
 167         int ret, do_wakeup, page_nr;
 168
 169         ret = 0;
 170         do_wakeup = 0;
 171         page_nr = 0;
 172
 173         if (pipe->inode)
 174                 mutex_lock(&pipe->inode->i_mutex);
 175
 176         for (;;) {
 177                 if (!pipe->readers) {
 178                         send_sig(SIGPIPE, current, 0);
 179                         if (!ret)
 180                                 ret = -EPIPE;
 181                         break;
 182                 }
 183
 184                 if (pipe->nrbufs < PIPE_BUFFERS) {
 185                         int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
 186                         struct pipe_buffer *buf = pipe->bufs + newbuf;
 187
 188                         buf->page = spd->pages[page_nr];
 189                         buf->offset = spd->partial[page_nr].offset;
 190                         buf->len = spd->partial[page_nr].len;
 191                         buf->ops = spd->ops;
 192                         if (spd->flags & SPLICE_F_GIFT)
 193                                 buf->flags |= PIPE_BUF_FLAG_GIFT;
 194
 195                         pipe->nrbufs++;
 196                         page_nr++;
 197                         ret += buf->len;
 198
 199                         if (pipe->inode)
 200                                 do_wakeup = 1;
 201
 202                         if (!--spd->nr_pages)
 203                                 break;
 204                         if (pipe->nrbufs < PIPE_BUFFERS)
 205                                 continue;
 206
 207                         break;
 208                 }
 209
 210                 if (spd->flags & SPLICE_F_NONBLOCK) {
 211                         if (!ret)
 212                                 ret = -EAGAIN;
 213                         break;
 214                 }
 215
 216                 if (signal_pending(current)) {
 217                         if (!ret)
 218                                 ret = -ERESTARTSYS;
 219                         break;
 220                 }
 221
 222                 if (do_wakeup) {
 223                         smp_mb();
 224                         if (waitqueue_active(&pipe->wait))
 225                                 wake_up_interruptible_sync(&pipe->wait);
 226                         kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 227                         do_wakeup = 0;
 228                 }
 229
 230                 pipe->waiting_writers++;
 231                 pipe_wait(pipe);
 232                 pipe->waiting_writers--;
 233         }
 234
 235         if (pipe->inode)
 236                 mutex_unlock(&pipe->inode->i_mutex);
 237
 238         if (do_wakeup) {
 239                 smp_mb();
 240                 if (waitqueue_active(&pipe->wait))
 241                         wake_up_interruptible(&pipe->wait);
 242                 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 243         }
 244
 245         while (page_nr < spd->nr_pages)
 246                 page_cache_release(spd->pages[page_nr++]);
 247
 248         return ret;
 249 }
 250
 251 static int
 252 __generic_file_splice_read(struct file *in, loff_t *ppos,
 253                            struct pipe_inode_info *pipe, size_t len,
 254                            unsigned int flags)
 255 {
 256         struct address_space *mapping = in->f_mapping;
 257         unsigned int loff, nr_pages;
 258         struct page *pages[PIPE_BUFFERS];
 259         struct partial_page partial[PIPE_BUFFERS];
 260         struct page *page;
 261         pgoff_t index, end_index;
 262         loff_t isize;
 263         size_t total_len;
 264         int error, page_nr;
 265         struct splice_pipe_desc spd = {
 266                 .pages = pages,
 267                 .partial = partial,
 268                 .flags = flags,
 269                 .ops = &page_cache_pipe_buf_ops,
 270         };
 271
 272         index = *ppos >> PAGE_CACHE_SHIFT;
 273         loff = *ppos & ~PAGE_CACHE_MASK;
 274         nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 275
 276         if (nr_pages > PIPE_BUFFERS)
 277                 nr_pages = PIPE_BUFFERS;
 278
 279         /*
 280          * Initiate read-ahead on this page range. however, don't call into
 281          * read-ahead if this is a non-zero offset (we are likely doing small
 282          * chunk splice and the page is already there) for a single page.
 283          */
 284         if (!loff || nr_pages > 1)
 285                 page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
 286
 287         /*
 288          * Now fill in the holes:
 289          */
 290         error = 0;
 291         total_len = 0;
 292
 293         /*
 294          * Lookup the (hopefully) full range of pages we need.
 295          */
 296         spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
 297
 298         /*
 299          * If find_get_pages_contig() returned fewer pages than we needed,
 300          * allocate the rest.
 301          */
 302         index += spd.nr_pages;
 303         while (spd.nr_pages < nr_pages) {
 304                 /*
 305                  * Page could be there, find_get_pages_contig() breaks on
 306                  * the first hole.
 307                  */
 308                 page = find_get_page(mapping, index);
 309                 if (!page) {
 310                         /*
 311                          * Make sure the read-ahead engine is notified
 312                          * about this failure.
 313                          */
 314                         handle_ra_miss(mapping, &in->f_ra, index);
 315
 316                         /*
 317                          * page didn't exist, allocate one.
 318                          */
 319                         page = page_cache_alloc_cold(mapping);
 320                         if (!page)
 321                                 break;
 322
 323                         error = add_to_page_cache_lru(page, mapping, index,
 324                                               mapping_gfp_mask(mapping));
 325                         if (unlikely(error)) {
 326                                 page_cache_release(page);
 327                                 if (error == -EEXIST)
 328                                         continue;
 329                                 break;
 330                         }
 331                         /*
 332                          * add_to_page_cache() locks the page, unlock it
 333                          * to avoid convoluting the logic below even more.
 334                          */
 335                         unlock_page(page);
 336                 }
 337
 338                 pages[spd.nr_pages++] = page;
 339                 index++;
 340         }
 341
 342         /*
 343          * Now loop over the map and see if we need to start IO on any
 344          * pages, fill in the partial map, etc.
 345          */
 346         index = *ppos >> PAGE_CACHE_SHIFT;
 347         nr_pages = spd.nr_pages;
 348         spd.nr_pages = 0;
 349         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
 350                 unsigned int this_len;
 351
 352                 if (!len)
 353                         break;
 354
 355                 /*
 356                  * this_len is the max we'll use from this page
 357                  */
 358                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
 359                 page = pages[page_nr];
 360
 361                 /*
 362                  * If the page isn't uptodate, we may need to start io on it
 363                  */
 364                 if (!PageUptodate(page)) {
 365                         /*
 366                          * If in nonblock mode then dont block on waiting
 367                          * for an in-flight io page
 368                          */
 369                         if (flags & SPLICE_F_NONBLOCK)
 370                                 break;
 371
 372                         lock_page(page);
 373
 374                         /*
 375                          * page was truncated, stop here. if this isn't the
 376                          * first page, we'll just complete what we already
 377                          * added
 378                          */
 379                         if (!page->mapping) {
 380                                 unlock_page(page);
 381                                 break;
 382                         }
 383                         /*
 384                          * page was already under io and is now done, great
 385                          */
 386                         if (PageUptodate(page)) {
 387                                 unlock_page(page);
 388                                 goto fill_it;
 389                         }
 390
 391                         /*
 392                          * need to read in the page
 393                          */
 394                         error = mapping->a_ops->readpage(in, page);
 395                         if (unlikely(error)) {
 396                                 /*
 397                                  * We really should re-lookup the page here,
 398                                  * but it complicates things a lot. Instead
 399                                  * lets just do what we already stored, and
 400                                  * we'll get it the next time we are called.
 401                                  */
 402                                 if (error == AOP_TRUNCATED_PAGE)
 403                                         error = 0;
 404
 405                                 break;
 406                         }
 407
 408                         /*
 409                          * i_size must be checked after ->readpage().
 410                          */
 411                         isize = i_size_read(mapping->host);
 412                         end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
 413                         if (unlikely(!isize || index > end_index))
 414                                 break;
 415
 416                         /*
 417                          * if this is the last page, see if we need to shrink
 418                          * the length and stop
 419                          */
 420                         if (end_index == index) {
 421                                 loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
 422                                 if (total_len + loff > isize)
 423                                         break;
 424                                 /*
 425                                  * force quit after adding this page
 426                                  */
 427                                 len = this_len;
 428                                 this_len = min(this_len, loff);
 429                                 loff = 0;
 430                         }
 431                 }
 432 fill_it:
 433                 partial[page_nr].offset = loff;
 434                 partial[page_nr].len = this_len;
 435                 len -= this_len;
 436                 total_len += this_len;
 437                 loff = 0;
 438                 spd.nr_pages++;
 439                 index++;
 440         }
 441
 442         /*
 443          * Release any pages at the end, if we quit early. 'i' is how far
 444          * we got, 'nr_pages' is how many pages are in the map.
 445          */
 446         while (page_nr < nr_pages)
 447                 page_cache_release(pages[page_nr++]);
 448
 449         if (spd.nr_pages)
 450                 return splice_to_pipe(pipe, &spd);
 451
 452         return error;
 453 }
 454
 455 /**
 456  * generic_file_splice_read - splice data from file to a pipe
 457  * @in:         file to splice from
 458  * @pipe:       pipe to splice to
 459  * @len:        number of bytes to splice
 460  * @flags:      splice modifier flags
 461  *
 462  * Will read pages from given file and fill them into a pipe.
 463  */
 464 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
 465                                  struct pipe_inode_info *pipe, size_t len,
 466                                  unsigned int flags)
 467 {
 468         ssize_t spliced;
 469         int ret;
 470
 471         ret = 0;
 472         spliced = 0;
 473
 474         while (len) {
 475                 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
 476
 477                 if (ret < 0)
 478                         break;
 479                 else if (!ret) {
 480                         if (spliced)
 481                                 break;
 482                         if (flags & SPLICE_F_NONBLOCK) {
 483                                 ret = -EAGAIN;
 484                                 break;
 485                         }
 486                 }
 487
 488                 *ppos += ret;
 489                 len -= ret;
 490                 spliced += ret;
 491         }
 492
 493         if (spliced)
 494                 return spliced;
 495
 496         return ret;
 497 }
 498
 499 EXPORT_SYMBOL(generic_file_splice_read);
 500
 501 /*
 502  * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
 503  * using sendpage(). Return the number of bytes sent.
 504  */
 505 static int pipe_to_sendpage(struct pipe_inode_info *pipe,
 506                             struct pipe_buffer *buf, struct splice_desc *sd)
 507 {
 508         struct file *file = sd->file;
 509         loff_t pos = sd->pos;
 510         int ret, more;
 511
 512         ret = buf->ops->pin(pipe, buf);
 513         if (!ret) {
 514                 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
 515
 516                 ret = file->f_op->sendpage(file, buf->page, buf->offset,
 517                                            sd->len, &pos, more);
 518         }
 519
 520         return ret;
 521 }
 522
 523 /*
 524  * This is a little more tricky than the file -> pipe splicing. There are
 525  * basically three cases:
 526  *
 527  *      - Destination page already exists in the address space and there
 528  *        are users of it. For that case we have no other option that
 529  *        copying the data. Tough luck.
 530  *      - Destination page already exists in the address space, but there
 531  *        are no users of it. Make sure it's uptodate, then drop it. Fall
 532  *        through to last case.
 533  *      - Destination page does not exist, we can add the pipe page to
 534  *        the page cache and avoid the copy.
 535  *
 536  * If asked to move pages to the output file (SPLICE_F_MOVE is set in
 537  * sd->flags), we attempt to migrate pages from the pipe to the output
 538  * file address space page cache. This is possible if no one else has
 539  * the pipe page referenced outside of the pipe and page cache. If
 540  * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
 541  * a new page in the output file page cache and fill/dirty that.
 542  */
 543 static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
 544                         struct splice_desc *sd)
 545 {
 546         struct file *file = sd->file;
 547         struct address_space *mapping = file->f_mapping;
 548         gfp_t gfp_mask = mapping_gfp_mask(mapping);
 549         unsigned int offset, this_len;
 550         struct page *page;
 551         pgoff_t index;
 552         int ret;
 553
 554         /*
 555          * make sure the data in this buffer is uptodate
 556          */
 557         ret = buf->ops->pin(pipe, buf);
 558         if (unlikely(ret))
 559                 return ret;
 560
 561         index = sd->pos >> PAGE_CACHE_SHIFT;
 562         offset = sd->pos & ~PAGE_CACHE_MASK;
 563
 564         this_len = sd->len;
 565         if (this_len + offset > PAGE_CACHE_SIZE)
 566                 this_len = PAGE_CACHE_SIZE - offset;
 567
 568         /*
 569          * Reuse buf page, if SPLICE_F_MOVE is set and we are doing a full
 570          * page.
 571          */
 572         if ((sd->flags & SPLICE_F_MOVE) && this_len == PAGE_CACHE_SIZE) {
 573                 /*
 574                  * If steal succeeds, buf->page is now pruned from the
 575                  * pagecache and we can reuse it. The page will also be
 576                  * locked on successful return.
 577                  */
 578                 if (buf->ops->steal(pipe, buf))
 579                         goto find_page;
 580
 581                 page = buf->page;
 582                 if (add_to_page_cache(page, mapping, index, gfp_mask)) {
 583                         unlock_page(page);
 584                         goto find_page;
 585                 }
 586
 587                 page_cache_get(page);
 588
 589                 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
 590                         lru_cache_add(page);
 591         } else {
 592 find_page:
 593                 page = find_lock_page(mapping, index);
 594                 if (!page) {
 595                         ret = -ENOMEM;
 596                         page = page_cache_alloc_cold(mapping);
 597                         if (unlikely(!page))
 598                                 goto out_nomem;
 599
 600                         /*
 601                          * This will also lock the page
 602                          */
 603                         ret = add_to_page_cache_lru(page, mapping, index,
 604                                                     gfp_mask);
 605                         if (unlikely(ret))
 606                                 goto out;
 607                 }
 608
 609                 /*
 610                  * We get here with the page locked. If the page is also
 611                  * uptodate, we don't need to do more. If it isn't, we
 612                  * may need to bring it in if we are not going to overwrite
 613                  * the full page.
 614                  */
 615                 if (!PageUptodate(page)) {
 616                         if (this_len < PAGE_CACHE_SIZE) {
 617                                 ret = mapping->a_ops->readpage(file, page);
 618                                 if (unlikely(ret))
 619                                         goto out;
 620
 621                                 lock_page(page);
 622
 623                                 if (!PageUptodate(page)) {
 624                                         /*
 625                                          * Page got invalidated, repeat.
 626                                          */
 627                                         if (!page->mapping) {
 628                                                 unlock_page(page);
 629                                                 page_cache_release(page);
 630                                                 goto find_page;
 631                                         }
 632                                         ret = -EIO;
 633                                         goto out;
 634                                 }
 635                         } else
 636                                 SetPageUptodate(page);
 637                 }
 638         }
 639
 640         ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
 641         if (unlikely(ret)) {
 642                 loff_t isize = i_size_read(mapping->host);
 643
 644                 if (ret != AOP_TRUNCATED_PAGE)
 645                         unlock_page(page);
 646                 page_cache_release(page);
 647                 if (ret == AOP_TRUNCATED_PAGE)
 648                         goto find_page;
 649
 650                 /*
 651                  * prepare_write() may have instantiated a few blocks
 652                  * outside i_size.  Trim these off again.
 653                  */
 654                 if (sd->pos + this_len > isize)
 655                         vmtruncate(mapping->host, isize);
 656
 657                 goto out;
 658         }
 659
 660         if (buf->page != page) {
 661                 /*
 662                  * Careful, ->map() uses KM_USER0!
 663                  */
 664                 char *src = buf->ops->map(pipe, buf, 1);
 665                 char *dst = kmap_atomic(page, KM_USER1);
 666
 667                 memcpy(dst + offset, src + buf->offset, this_len);
 668                 flush_dcache_page(page);
 669                 kunmap_atomic(dst, KM_USER1);
 670                 buf->ops->unmap(pipe, buf, src);
 671         }
 672
 673         ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
 674         if (!ret) {
 675                 /*
 676                  * Return the number of bytes written and mark page as
 677                  * accessed, we are now done!
 678                  */
 679                 ret = this_len;
 680                 mark_page_accessed(page);
 681                 balance_dirty_pages_ratelimited(mapping);
 682         } else if (ret == AOP_TRUNCATED_PAGE) {
 683                 page_cache_release(page);
 684                 goto find_page;
 685         }
 686 out:
 687         page_cache_release(page);
 688         unlock_page(page);
 689 out_nomem:
 690         return ret;
 691 }
 692
 693 /*
 694  * Pipe input worker. Most of this logic works like a regular pipe, the
 695  * key here is the 'actor' worker passed in that actually moves the data
 696  * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
 697  */
 698 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
 699                          loff_t *ppos, size_t len, unsigned int flags,
 700                          splice_actor *actor)
 701 {
 702         int ret, do_wakeup, err;
 703         struct splice_desc sd;
 704
 705         ret = 0;
 706         do_wakeup = 0;
 707
 708         sd.total_len = len;
 709         sd.flags = flags;
 710         sd.file = out;
 711         sd.pos = *ppos;
 712
 713         if (pipe->inode)
 714                 mutex_lock(&pipe->inode->i_mutex);
 715
 716         for (;;) {
 717                 if (pipe->nrbufs) {
 718                         struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
 719                         struct pipe_buf_operations *ops = buf->ops;
 720
 721                         sd.len = buf->len;
 722                         if (sd.len > sd.total_len)
 723                                 sd.len = sd.total_len;
 724
 725                         err = actor(pipe, buf, &sd);
 726                         if (err <= 0) {
 727                                 if (!ret && err != -ENODATA)
 728                                         ret = err;
 729
 730                                 break;
 731                         }
 732
 733                         ret += err;
 734                         buf->offset += err;
 735                         buf->len -= err;
 736
 737                         sd.len -= err;
 738                         sd.pos += err;
 739                         sd.total_len -= err;
 740                         if (sd.len)
 741                                 continue;
 742
 743                         if (!buf->len) {
 744                                 buf->ops = NULL;
 745                                 ops->release(pipe, buf);
 746                                 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
 747                                 pipe->nrbufs--;
 748                                 if (pipe->inode)
 749                                         do_wakeup = 1;
 750                         }
 751
 752                         if (!sd.total_len)
 753                                 break;
 754                 }
 755
 756                 if (pipe->nrbufs)
 757                         continue;
 758                 if (!pipe->writers)
 759                         break;
 760                 if (!pipe->waiting_writers) {
 761                         if (ret)
 762                                 break;
 763                 }
 764
 765                 if (flags & SPLICE_F_NONBLOCK) {
 766                         if (!ret)
 767                                 ret = -EAGAIN;
 768                         break;
 769                 }
 770
 771                 if (signal_pending(current)) {
 772                         if (!ret)
 773                                 ret = -ERESTARTSYS;
 774                         break;
 775                 }
 776
 777                 if (do_wakeup) {
 778                         smp_mb();
 779                         if (waitqueue_active(&pipe->wait))
 780                                 wake_up_interruptible_sync(&pipe->wait);
 781                         kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 782                         do_wakeup = 0;
 783                 }
 784
 785                 pipe_wait(pipe);
 786         }
 787
 788         if (pipe->inode)
 789                 mutex_unlock(&pipe->inode->i_mutex);
 790
 791         if (do_wakeup) {
 792                 smp_mb();
 793                 if (waitqueue_active(&pipe->wait))
 794                         wake_up_interruptible(&pipe->wait);
 795                 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 796         }
 797
 798         return ret;
 799 }
 800
 801 /**
 802  * generic_file_splice_write - splice data from a pipe to a file
 803  * @pipe:       pipe info
 804  * @out:        file to write to
 805  * @len:        number of bytes to splice
 806  * @flags:      splice modifier flags
 807  *
 808  * Will either move or copy pages (determined by @flags options) from
 809  * the given pipe inode to the given file.
 810  *
 811  */
 812 ssize_t
 813 generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
 814                           loff_t *ppos, size_t len, unsigned int flags)
 815 {
 816         struct address_space *mapping = out->f_mapping;
 817         ssize_t ret;
 818
 819         ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
 820         if (ret > 0) {
 821                 struct inode *inode = mapping->host;
 822
 823                 *ppos += ret;
 824
 825                 /*
 826                  * If file or inode is SYNC and we actually wrote some data,
 827                  * sync it.
 828                  */
 829                 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
 830                         int err;
 831
 832                         mutex_lock(&inode->i_mutex);
 833                         err = generic_osync_inode(inode, mapping,
 834                                                   OSYNC_METADATA|OSYNC_DATA);
 835                         mutex_unlock(&inode->i_mutex);
 836
 837                         if (err)
 838                                 ret = err;
 839                 }
 840         }
 841
 842         return ret;
 843 }
 844
 845 EXPORT_SYMBOL(generic_file_splice_write);
 846
 847 /**
 848  * generic_splice_sendpage - splice data from a pipe to a socket
 849  * @inode:      pipe inode
 850  * @out:        socket to write to
 851  * @len:        number of bytes to splice
 852  * @flags:      splice modifier flags
 853  *
 854  * Will send @len bytes from the pipe to a network socket. No data copying
 855  * is involved.
 856  *
 857  */
 858 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
 859                                 loff_t *ppos, size_t len, unsigned int flags)
 860 {
 861         return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
 862 }
 863
 864 EXPORT_SYMBOL(generic_splice_sendpage);
 865
 866 /*
 867  * Attempt to initiate a splice from pipe to file.
 868  */
 869 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
 870                            loff_t *ppos, size_t len, unsigned int flags)
 871 {
 872         int ret;
 873
 874         if (unlikely(!out->f_op || !out->f_op->splice_write))
 875                 return -EINVAL;
 876
 877         if (unlikely(!(out->f_mode & FMODE_WRITE)))
 878                 return -EBADF;
 879
 880         ret = rw_verify_area(WRITE, out, ppos, len);
 881         if (unlikely(ret < 0))
 882                 return ret;
 883
 884         return out->f_op->splice_write(pipe, out, ppos, len, flags);
 885 }
 886
 887 /*
 888  * Attempt to initiate a splice from a file to a pipe.
 889  */
 890 static long do_splice_to(struct file *in, loff_t *ppos,
 891                          struct pipe_inode_info *pipe, size_t len,
 892                          unsigned int flags)
 893 {
 894         loff_t isize, left;
 895         int ret;
 896
 897         if (unlikely(!in->f_op || !in->f_op->splice_read))
 898                 return -EINVAL;
 899
 900         if (unlikely(!(in->f_mode & FMODE_READ)))
 901                 return -EBADF;
 902
 903         ret = rw_verify_area(READ, in, ppos, len);
 904         if (unlikely(ret < 0))
 905                 return ret;
 906
 907         isize = i_size_read(in->f_mapping->host);
 908         if (unlikely(*ppos >= isize))
 909                 return 0;
 910
 911         left = isize - *ppos;
 912         if (unlikely(left < len))
 913                 len = left;
 914
 915         return in->f_op->splice_read(in, ppos, pipe, len, flags);
 916 }
 917
 918 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
 919                       size_t len, unsigned int flags)
 920 {
 921         struct pipe_inode_info *pipe;
 922         long ret, bytes;
 923         loff_t out_off;
 924         umode_t i_mode;
 925         int i;
 926
 927         /*
 928          * We require the input being a regular file, as we don't want to
 929          * randomly drop data for eg socket -> socket splicing. Use the
 930          * piped splicing for that!
 931          */
 932         i_mode = in->f_dentry->d_inode->i_mode;
 933         if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
 934                 return -EINVAL;
 935
 936         /*
 937          * neither in nor out is a pipe, setup an internal pipe attached to
 938          * 'out' and transfer the wanted data from 'in' to 'out' through that
 939          */
 940         pipe = current->splice_pipe;
 941         if (unlikely(!pipe)) {
 942                 pipe = alloc_pipe_info(NULL);
 943                 if (!pipe)
 944                         return -ENOMEM;
 945
 946                 /*
 947                  * We don't have an immediate reader, but we'll read the stuff
 948                  * out of the pipe right after the splice_to_pipe(). So set
 949                  * PIPE_READERS appropriately.
 950                  */
 951                 pipe->readers = 1;
 952
 953                 current->splice_pipe = pipe;
 954         }
 955
 956         /*
 957          * Do the splice.
 958          */
 959         ret = 0;
 960         bytes = 0;
 961         out_off = 0;
 962
 963         while (len) {
 964                 size_t read_len, max_read_len;
 965
 966                 /*
 967                  * Do at most PIPE_BUFFERS pages worth of transfer:
 968                  */
 969                 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
 970
 971                 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
 972                 if (unlikely(ret < 0))
 973                         goto out_release;
 974
 975                 read_len = ret;
 976
 977                 /*
 978                  * NOTE: nonblocking mode only applies to the input. We
 979                  * must not do the output in nonblocking mode as then we
 980                  * could get stuck data in the internal pipe:
 981                  */
 982                 ret = do_splice_from(pipe, out, &out_off, read_len,
 983                                      flags & ~SPLICE_F_NONBLOCK);
 984                 if (unlikely(ret < 0))
 985                         goto out_release;
 986
 987                 bytes += ret;
 988                 len -= ret;
 989
 990                 /*
 991                  * In nonblocking mode, if we got back a short read then
 992                  * that was due to either an IO error or due to the
 993                  * pagecache entry not being there. In the IO error case
 994                  * the _next_ splice attempt will produce a clean IO error
 995                  * return value (not a short read), so in both cases it's
 996                  * correct to break out of the loop here:
 997                  */
 998                 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
 999                         break;
1000         }
1001
1002         pipe->nrbufs = pipe->curbuf = 0;
1003
1004         return bytes;
1005
1006 out_release:
1007         /*
1008          * If we did an incomplete transfer we must release
1009          * the pipe buffers in question:
1010          */
1011         for (i = 0; i < PIPE_BUFFERS; i++) {
1012                 struct pipe_buffer *buf = pipe->bufs + i;
1013
1014                 if (buf->ops) {
1015                         buf->ops->release(pipe, buf);
1016                         buf->ops = NULL;
1017                 }
1018         }
1019         pipe->nrbufs = pipe->curbuf = 0;
1020
1021         /*
1022          * If we transferred some data, return the number of bytes:
1023          */
1024         if (bytes > 0)
1025                 return bytes;
1026
1027         return ret;
1028 }
1029
1030 EXPORT_SYMBOL(do_splice_direct);
1031
1032 /*
1033  * Determine where to splice to/from.
1034  */
1035 static long do_splice(struct file *in, loff_t __user *off_in,
1036                       struct file *out, loff_t __user *off_out,
1037                       size_t len, unsigned int flags)
1038 {
1039         struct pipe_inode_info *pipe;
1040         loff_t offset, *off;
1041         long ret;
1042
1043         pipe = in->f_dentry->d_inode->i_pipe;
1044         if (pipe) {
1045                 if (off_in)
1046                         return -ESPIPE;
1047                 if (off_out) {
1048                         if (out->f_op->llseek == no_llseek)
1049                                 return -EINVAL;
1050                         if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1051                                 return -EFAULT;
1052                         off = &offset;
1053                 } else
1054                         off = &out->f_pos;
1055
1056                 ret = do_splice_from(pipe, out, off, len, flags);
1057
1058                 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1059                         ret = -EFAULT;
1060
1061                 return ret;
1062         }
1063
1064         pipe = out->f_dentry->d_inode->i_pipe;
1065         if (pipe) {
1066                 if (off_out)
1067                         return -ESPIPE;
1068                 if (off_in) {
1069                         if (in->f_op->llseek == no_llseek)
1070                                 return -EINVAL;
1071                         if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1072                                 return -EFAULT;
1073                         off = &offset;
1074                 } else
1075                         off = &in->f_pos;
1076
1077                 ret = do_splice_to(in, off, pipe, len, flags);
1078
1079                 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1080                         ret = -EFAULT;
1081
1082                 return ret;
1083         }
1084
1085         return -EINVAL;
1086 }
1087
1088 /*
1089  * Map an iov into an array of pages and offset/length tupples. With the
1090  * partial_page structure, we can map several non-contiguous ranges into
1091  * our ones pages[] map instead of splitting that operation into pieces.
1092  * Could easily be exported as a generic helper for other users, in which
1093  * case one would probably want to add a 'max_nr_pages' parameter as well.
1094  */
1095 static int get_iovec_page_array(const struct iovec __user *iov,
1096                                 unsigned int nr_vecs, struct page **pages,
1097                                 struct partial_page *partial, int aligned)
1098 {
1099         int buffers = 0, error = 0;
1100
1101         /*
1102          * It's ok to take the mmap_sem for reading, even
1103          * across a "get_user()".
1104          */
1105         down_read(&current->mm->mmap_sem);
1106
1107         while (nr_vecs) {
1108                 unsigned long off, npages;
1109                 void __user *base;
1110                 size_t len;
1111                 int i;
1112
1113                 /*
1114                  * Get user address base and length for this iovec.
1115                  */
1116                 error = get_user(base, &iov->iov_base);
1117                 if (unlikely(error))
1118                         break;
1119                 error = get_user(len, &iov->iov_len);
1120                 if (unlikely(error))
1121                         break;
1122
1123                 /*
1124                  * Sanity check this iovec. 0 read succeeds.
1125                  */
1126                 if (unlikely(!len))
1127                         break;
1128                 error = -EFAULT;
1129                 if (unlikely(!base))
1130                         break;
1131
1132                 /*
1133                  * Get this base offset and number of pages, then map
1134                  * in the user pages.
1135                  */
1136                 off = (unsigned long) base & ~PAGE_MASK;
1137
1138                 /*
1139                  * If asked for alignment, the offset must be zero and the
1140                  * length a multiple of the PAGE_SIZE.
1141                  */
1142                 error = -EINVAL;
1143                 if (aligned && (off || len & ~PAGE_MASK))
1144                         break;
1145
1146                 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1147                 if (npages > PIPE_BUFFERS - buffers)
1148                         npages = PIPE_BUFFERS - buffers;
1149
1150                 error = get_user_pages(current, current->mm,
1151                                        (unsigned long) base, npages, 0, 0,
1152                                        &pages[buffers], NULL);
1153
1154                 if (unlikely(error <= 0))
1155                         break;
1156
1157                 /*
1158                  * Fill this contiguous range into the partial page map.
1159                  */
1160                 for (i = 0; i < error; i++) {
1161                         const int plen = min_t(size_t, len, PAGE_SIZE - off);
1162
1163                         partial[buffers].offset = off;
1164                         partial[buffers].len = plen;
1165
1166                         off = 0;
1167                         len -= plen;
1168                         buffers++;
1169                 }
1170
1171                 /*
1172                  * We didn't complete this iov, stop here since it probably
1173                  * means we have to move some of this into a pipe to
1174                  * be able to continue.
1175                  */
1176                 if (len)
1177                         break;
1178
1179                 /*
1180                  * Don't continue if we mapped fewer pages than we asked for,
1181                  * or if we mapped the max number of pages that we have
1182                  * room for.
1183                  */
1184                 if (error < npages || buffers == PIPE_BUFFERS)
1185                         break;
1186
1187                 nr_vecs--;
1188                 iov++;
1189         }
1190
1191         up_read(&current->mm->mmap_sem);
1192
1193         if (buffers)
1194                 return buffers;
1195
1196         return error;
1197 }
1198
1199 /*
1200  * vmsplice splices a user address range into a pipe. It can be thought of
1201  * as splice-from-memory, where the regular splice is splice-from-file (or
1202  * to file). In both cases the output is a pipe, naturally.
1203  *
1204  * Note that vmsplice only supports splicing _from_ user memory to a pipe,
1205  * not the other way around. Splicing from user memory is a simple operation
1206  * that can be supported without any funky alignment restrictions or nasty
1207  * vm tricks. We simply map in the user memory and fill them into a pipe.
1208  * The reverse isn't quite as easy, though. There are two possible solutions
1209  * for that:
1210  *
1211  *      - memcpy() the data internally, at which point we might as well just
1212  *        do a regular read() on the buffer anyway.
1213  *      - Lots of nasty vm tricks, that are neither fast nor flexible (it
1214  *        has restriction limitations on both ends of the pipe).
1215  *
1216  * Alas, it isn't here.
1217  *
1218  */
1219 static long do_vmsplice(struct file *file, const struct iovec __user *iov,
1220                         unsigned long nr_segs, unsigned int flags)
1221 {
1222         struct pipe_inode_info *pipe = file->f_dentry->d_inode->i_pipe;
1223         struct page *pages[PIPE_BUFFERS];
1224         struct partial_page partial[PIPE_BUFFERS];
1225         struct splice_pipe_desc spd = {
1226                 .pages = pages,
1227                 .partial = partial,
1228                 .flags = flags,
1229                 .ops = &user_page_pipe_buf_ops,
1230         };
1231
1232         if (unlikely(!pipe))
1233                 return -EBADF;
1234         if (unlikely(nr_segs > UIO_MAXIOV))
1235                 return -EINVAL;
1236         else if (unlikely(!nr_segs))
1237                 return 0;
1238
1239         spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial,
1240                                             flags & SPLICE_F_GIFT);
1241         if (spd.nr_pages <= 0)
1242                 return spd.nr_pages;
1243
1244         return splice_to_pipe(pipe, &spd);
1245 }
1246
1247 asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov,
1248                              unsigned long nr_segs, unsigned int flags)
1249 {
1250         struct file *file;
1251         long error;
1252         int fput;
1253
1254         error = -EBADF;
1255         file = fget_light(fd, &fput);
1256         if (file) {
1257                 if (file->f_mode & FMODE_WRITE)
1258                         error = do_vmsplice(file, iov, nr_segs, flags);
1259
1260                 fput_light(file, fput);
1261         }
1262
1263         return error;
1264 }
1265
1266 asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
1267                            int fd_out, loff_t __user *off_out,
1268                            size_t len, unsigned int flags)
1269 {
1270         long error;
1271         struct file *in, *out;
1272         int fput_in, fput_out;
1273
1274         if (unlikely(!len))
1275                 return 0;
1276
1277         error = -EBADF;
1278         in = fget_light(fd_in, &fput_in);
1279         if (in) {
1280                 if (in->f_mode & FMODE_READ) {
1281                         out = fget_light(fd_out, &fput_out);
1282                         if (out) {
1283                                 if (out->f_mode & FMODE_WRITE)
1284                                         error = do_splice(in, off_in,
1285                                                           out, off_out,
1286                                                           len, flags);
1287                                 fput_light(out, fput_out);
1288                         }
1289                 }
1290
1291                 fput_light(in, fput_in);
1292         }
1293
1294         return error;
1295 }
1296
1297 /*
1298  * Link contents of ipipe to opipe.
1299  */
1300 static int link_pipe(struct pipe_inode_info *ipipe,
1301                      struct pipe_inode_info *opipe,
1302                      size_t len, unsigned int flags)
1303 {
1304         struct pipe_buffer *ibuf, *obuf;
1305         int ret, do_wakeup, i, ipipe_first;
1306
1307         ret = do_wakeup = ipipe_first = 0;
1308
1309         /*
1310          * Potential ABBA deadlock, work around it by ordering lock
1311          * grabbing by inode address. Otherwise two different processes
1312          * could deadlock (one doing tee from A -> B, the other from B -> A).
1313          */
1314         if (ipipe->inode < opipe->inode) {
1315                 ipipe_first = 1;
1316                 mutex_lock(&ipipe->inode->i_mutex);
1317                 mutex_lock(&opipe->inode->i_mutex);
1318         } else {
1319                 mutex_lock(&opipe->inode->i_mutex);
1320                 mutex_lock(&ipipe->inode->i_mutex);
1321         }
1322
1323         for (i = 0;; i++) {
1324                 if (!opipe->readers) {
1325                         send_sig(SIGPIPE, current, 0);
1326                         if (!ret)
1327                                 ret = -EPIPE;
1328                         break;
1329                 }
1330                 if (ipipe->nrbufs - i) {
1331                         ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1332
1333                         /*
1334                          * If we have room, fill this buffer
1335                          */
1336                         if (opipe->nrbufs < PIPE_BUFFERS) {
1337                                 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1338
1339                                 /*
1340                                  * Get a reference to this pipe buffer,
1341                                  * so we can copy the contents over.
1342                                  */
1343                                 ibuf->ops->get(ipipe, ibuf);
1344
1345                                 obuf = opipe->bufs + nbuf;
1346                                 *obuf = *ibuf;
1347
1348                                 /*
1349                                  * Don't inherit the gift flag, we need to
1350                                  * prevent multiple steals of this page.
1351                                  */
1352                                 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1353
1354                                 if (obuf->len > len)
1355                                         obuf->len = len;
1356
1357                                 opipe->nrbufs++;
1358                                 do_wakeup = 1;
1359                                 ret += obuf->len;
1360                                 len -= obuf->len;
1361
1362                                 if (!len)
1363                                         break;
1364                                 if (opipe->nrbufs < PIPE_BUFFERS)
1365                                         continue;
1366                         }
1367
1368                         /*
1369                          * We have input available, but no output room.
1370                          * If we already copied data, return that. If we
1371                          * need to drop the opipe lock, it must be ordered
1372                          * last to avoid deadlocks.
1373                          */
1374                         if ((flags & SPLICE_F_NONBLOCK) || !ipipe_first) {
1375                                 if (!ret)
1376                                         ret = -EAGAIN;
1377                                 break;
1378                         }
1379                         if (signal_pending(current)) {
1380                                 if (!ret)
1381                                         ret = -ERESTARTSYS;
1382                                 break;
1383                         }
1384                         if (do_wakeup) {
1385                                 smp_mb();
1386                                 if (waitqueue_active(&opipe->wait))
1387                                         wake_up_interruptible(&opipe->wait);
1388                                 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1389                                 do_wakeup = 0;
1390                         }
1391
1392                         opipe->waiting_writers++;
1393                         pipe_wait(opipe);
1394                         opipe->waiting_writers--;
1395                         continue;
1396                 }
1397
1398                 /*
1399                  * No input buffers, do the usual checks for available
1400                  * writers and blocking and wait if necessary
1401                  */
1402                 if (!ipipe->writers)
1403                         break;
1404                 if (!ipipe->waiting_writers) {
1405                         if (ret)
1406                                 break;
1407                 }
1408                 /*
1409                  * pipe_wait() drops the ipipe mutex. To avoid deadlocks
1410                  * with another process, we can only safely do that if
1411                  * the ipipe lock is ordered last.
1412                  */
1413                 if ((flags & SPLICE_F_NONBLOCK) || ipipe_first) {
1414                         if (!ret)
1415                                 ret = -EAGAIN;
1416                         break;
1417                 }
1418                 if (signal_pending(current)) {
1419                         if (!ret)
1420                                 ret = -ERESTARTSYS;
1421                         break;
1422                 }
1423
1424                 if (waitqueue_active(&ipipe->wait))
1425                         wake_up_interruptible_sync(&ipipe->wait);
1426                 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1427
1428                 pipe_wait(ipipe);
1429         }
1430
1431         mutex_unlock(&ipipe->inode->i_mutex);
1432         mutex_unlock(&opipe->inode->i_mutex);
1433
1434         if (do_wakeup) {
1435                 smp_mb();
1436                 if (waitqueue_active(&opipe->wait))
1437                         wake_up_interruptible(&opipe->wait);
1438                 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1439         }
1440
1441         return ret;
1442 }
1443
1444 /*
1445  * This is a tee(1) implementation that works on pipes. It doesn't copy
1446  * any data, it simply references the 'in' pages on the 'out' pipe.
1447  * The 'flags' used are the SPLICE_F_* variants, currently the only
1448  * applicable one is SPLICE_F_NONBLOCK.
1449  */
1450 static long do_tee(struct file *in, struct file *out, size_t len,
1451                    unsigned int flags)
1452 {
1453         struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1454         struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1455
1456         /*
1457          * Link ipipe to the two output pipes, consuming as we go along.
1458          */
1459         if (ipipe && opipe)
1460                 return link_pipe(ipipe, opipe, len, flags);
1461
1462         return -EINVAL;
1463 }
1464
1465 asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1466 {
1467         struct file *in;
1468         int error, fput_in;
1469
1470         if (unlikely(!len))
1471                 return 0;
1472
1473         error = -EBADF;
1474         in = fget_light(fdin, &fput_in);
1475         if (in) {
1476                 if (in->f_mode & FMODE_READ) {
1477                         int fput_out;
1478                         struct file *out = fget_light(fdout, &fput_out);
1479
1480                         if (out) {
1481                                 if (out->f_mode & FMODE_WRITE)
1482                                         error = do_tee(in, out, len, flags);
1483                                 fput_light(out, fput_out);
1484                         }
1485                 }
1486                 fput_light(in, fput_in);
1487         }
1488
1489         return error;
1490 }