SAFE public projects git trees. - safe/jmp/linux-2.6/blob - fs/splice.c

   1 /*
   2  * "splice": joining two ropes together by interweaving their strands.
   3  *
   4  * This is the "extended pipe" functionality, where a pipe is used as
   5  * an arbitrary in-memory buffer. Think of a pipe as a small kernel
   6  * buffer that you can use to transfer data from one end to the other.
   7  *
   8  * The traditional unix read/write is extended with a "splice()" operation
   9  * that transfers data buffers to or from a pipe buffer.
  10  *
  11  * Named by Larry McVoy, original implementation from Linus, extended by
  12  * Jens to support splicing to files, network, direct splicing, etc and
  13  * fixing lots of bugs.
  14  *
  15  * Copyright (C) 2005-2006 Jens Axboe <axboe@suse.de>
  16  * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
  17  * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  18  *
  19  */
  20 #include <linux/fs.h>
  21 #include <linux/file.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/pipe_fs_i.h>
  24 #include <linux/mm_inline.h>
  25 #include <linux/swap.h>
  26 #include <linux/writeback.h>
  27 #include <linux/buffer_head.h>
  28 #include <linux/module.h>
  29 #include <linux/syscalls.h>
  30 #include <linux/uio.h>
  31
  32 struct partial_page {
  33         unsigned int offset;
  34         unsigned int len;
  35 };
  36
  37 /*
  38  * Passed to splice_to_pipe
  39  */
  40 struct splice_pipe_desc {
  41         struct page **pages;            /* page map */
  42         struct partial_page *partial;   /* pages[] may not be contig */
  43         int nr_pages;                   /* number of pages in map */
  44         unsigned int flags;             /* splice flags */
  45         struct pipe_buf_operations *ops;/* ops associated with output pipe */
  46 };
  47
  48 /*
  49  * Attempt to steal a page from a pipe buffer. This should perhaps go into
  50  * a vm helper function, it's already simplified quite a bit by the
  51  * addition of remove_mapping(). If success is returned, the caller may
  52  * attempt to reuse this page for another destination.
  53  */
  54 static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
  55                                      struct pipe_buffer *buf)
  56 {
  57         struct page *page = buf->page;
  58         struct address_space *mapping = page_mapping(page);
  59
  60         lock_page(page);
  61
  62         WARN_ON(!PageUptodate(page));
  63
  64         /*
  65          * At least for ext2 with nobh option, we need to wait on writeback
  66          * completing on this page, since we'll remove it from the pagecache.
  67          * Otherwise truncate wont wait on the page, allowing the disk
  68          * blocks to be reused by someone else before we actually wrote our
  69          * data to them. fs corruption ensues.
  70          */
  71         wait_on_page_writeback(page);
  72
  73         if (PagePrivate(page))
  74                 try_to_release_page(page, mapping_gfp_mask(mapping));
  75
  76         if (!remove_mapping(mapping, page)) {
  77                 unlock_page(page);
  78                 return 1;
  79         }
  80
  81         buf->flags |= PIPE_BUF_FLAG_LRU;
  82         return 0;
  83 }
  84
  85 static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
  86                                         struct pipe_buffer *buf)
  87 {
  88         page_cache_release(buf->page);
  89         buf->page = NULL;
  90         buf->flags &= ~PIPE_BUF_FLAG_LRU;
  91 }
  92
  93 static int page_cache_pipe_buf_pin(struct pipe_inode_info *info,
  94                                    struct pipe_buffer *buf)
  95 {
  96         struct page *page = buf->page;
  97         int err;
  98
  99         if (!PageUptodate(page)) {
 100                 lock_page(page);
 101
 102                 /*
 103                  * Page got truncated/unhashed. This will cause a 0-byte
 104                  * splice, if this is the first page.
 105                  */
 106                 if (!page->mapping) {
 107                         err = -ENODATA;
 108                         goto error;
 109                 }
 110
 111                 /*
 112                  * Uh oh, read-error from disk.
 113                  */
 114                 if (!PageUptodate(page)) {
 115                         err = -EIO;
 116                         goto error;
 117                 }
 118
 119                 /*
 120                  * Page is ok afterall, we are done.
 121                  */
 122                 unlock_page(page);
 123         }
 124
 125         return 0;
 126 error:
 127         unlock_page(page);
 128         return err;
 129 }
 130
 131 static struct pipe_buf_operations page_cache_pipe_buf_ops = {
 132         .can_merge = 0,
 133         .map = generic_pipe_buf_map,
 134         .unmap = generic_pipe_buf_unmap,
 135         .pin = page_cache_pipe_buf_pin,
 136         .release = page_cache_pipe_buf_release,
 137         .steal = page_cache_pipe_buf_steal,
 138         .get = generic_pipe_buf_get,
 139 };
 140
 141 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
 142                                     struct pipe_buffer *buf)
 143 {
 144         if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
 145                 return 1;
 146
 147         return 0;
 148 }
 149
 150 static struct pipe_buf_operations user_page_pipe_buf_ops = {
 151         .can_merge = 0,
 152         .map = generic_pipe_buf_map,
 153         .unmap = generic_pipe_buf_unmap,
 154         .pin = generic_pipe_buf_pin,
 155         .release = page_cache_pipe_buf_release,
 156         .steal = user_page_pipe_buf_steal,
 157         .get = generic_pipe_buf_get,
 158 };
 159
 160 /*
 161  * Pipe output worker. This sets up our pipe format with the page cache
 162  * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
 163  */
 164 static ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
 165                               struct splice_pipe_desc *spd)
 166 {
 167         int ret, do_wakeup, page_nr;
 168
 169         ret = 0;
 170         do_wakeup = 0;
 171         page_nr = 0;
 172
 173         if (pipe->inode)
 174                 mutex_lock(&pipe->inode->i_mutex);
 175
 176         for (;;) {
 177                 if (!pipe->readers) {
 178                         send_sig(SIGPIPE, current, 0);
 179                         if (!ret)
 180                                 ret = -EPIPE;
 181                         break;
 182                 }
 183
 184                 if (pipe->nrbufs < PIPE_BUFFERS) {
 185                         int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
 186                         struct pipe_buffer *buf = pipe->bufs + newbuf;
 187
 188                         buf->page = spd->pages[page_nr];
 189                         buf->offset = spd->partial[page_nr].offset;
 190                         buf->len = spd->partial[page_nr].len;
 191                         buf->ops = spd->ops;
 192                         if (spd->flags & SPLICE_F_GIFT)
 193                                 buf->flags |= PIPE_BUF_FLAG_GIFT;
 194
 195                         pipe->nrbufs++;
 196                         page_nr++;
 197                         ret += buf->len;
 198
 199                         if (pipe->inode)
 200                                 do_wakeup = 1;
 201
 202                         if (!--spd->nr_pages)
 203                                 break;
 204                         if (pipe->nrbufs < PIPE_BUFFERS)
 205                                 continue;
 206
 207                         break;
 208                 }
 209
 210                 if (spd->flags & SPLICE_F_NONBLOCK) {
 211                         if (!ret)
 212                                 ret = -EAGAIN;
 213                         break;
 214                 }
 215
 216                 if (signal_pending(current)) {
 217                         if (!ret)
 218                                 ret = -ERESTARTSYS;
 219                         break;
 220                 }
 221
 222                 if (do_wakeup) {
 223                         smp_mb();
 224                         if (waitqueue_active(&pipe->wait))
 225                                 wake_up_interruptible_sync(&pipe->wait);
 226                         kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 227                         do_wakeup = 0;
 228                 }
 229
 230                 pipe->waiting_writers++;
 231                 pipe_wait(pipe);
 232                 pipe->waiting_writers--;
 233         }
 234
 235         if (pipe->inode)
 236                 mutex_unlock(&pipe->inode->i_mutex);
 237
 238         if (do_wakeup) {
 239                 smp_mb();
 240                 if (waitqueue_active(&pipe->wait))
 241                         wake_up_interruptible(&pipe->wait);
 242                 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 243         }
 244
 245         while (page_nr < spd->nr_pages)
 246                 page_cache_release(spd->pages[page_nr++]);
 247
 248         return ret;
 249 }
 250
 251 static int
 252 __generic_file_splice_read(struct file *in, loff_t *ppos,
 253                            struct pipe_inode_info *pipe, size_t len,
 254                            unsigned int flags)
 255 {
 256         struct address_space *mapping = in->f_mapping;
 257         unsigned int loff, nr_pages;
 258         struct page *pages[PIPE_BUFFERS];
 259         struct partial_page partial[PIPE_BUFFERS];
 260         struct page *page;
 261         pgoff_t index, end_index;
 262         loff_t isize;
 263         size_t total_len;
 264         int error, page_nr;
 265         struct splice_pipe_desc spd = {
 266                 .pages = pages,
 267                 .partial = partial,
 268                 .flags = flags,
 269                 .ops = &page_cache_pipe_buf_ops,
 270         };
 271
 272         index = *ppos >> PAGE_CACHE_SHIFT;
 273         loff = *ppos & ~PAGE_CACHE_MASK;
 274         nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 275
 276         if (nr_pages > PIPE_BUFFERS)
 277                 nr_pages = PIPE_BUFFERS;
 278
 279         /*
 280          * Initiate read-ahead on this page range. however, don't call into
 281          * read-ahead if this is a non-zero offset (we are likely doing small
 282          * chunk splice and the page is already there) for a single page.
 283          */
 284         if (!loff || nr_pages > 1)
 285                 page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
 286
 287         /*
 288          * Now fill in the holes:
 289          */
 290         error = 0;
 291         total_len = 0;
 292
 293         /*
 294          * Lookup the (hopefully) full range of pages we need.
 295          */
 296         spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
 297
 298         /*
 299          * If find_get_pages_contig() returned fewer pages than we needed,
 300          * allocate the rest.
 301          */
 302         index += spd.nr_pages;
 303         while (spd.nr_pages < nr_pages) {
 304                 /*
 305                  * Page could be there, find_get_pages_contig() breaks on
 306                  * the first hole.
 307                  */
 308                 page = find_get_page(mapping, index);
 309                 if (!page) {
 310                         /*
 311                          * Make sure the read-ahead engine is notified
 312                          * about this failure.
 313                          */
 314                         handle_ra_miss(mapping, &in->f_ra, index);
 315
 316                         /*
 317                          * page didn't exist, allocate one.
 318                          */
 319                         page = page_cache_alloc_cold(mapping);
 320                         if (!page)
 321                                 break;
 322
 323                         error = add_to_page_cache_lru(page, mapping, index,
 324                                               mapping_gfp_mask(mapping));
 325                         if (unlikely(error)) {
 326                                 page_cache_release(page);
 327                                 break;
 328                         }
 329                         /*
 330                          * add_to_page_cache() locks the page, unlock it
 331                          * to avoid convoluting the logic below even more.
 332                          */
 333                         unlock_page(page);
 334                 }
 335
 336                 pages[spd.nr_pages++] = page;
 337                 index++;
 338         }
 339
 340         /*
 341          * Now loop over the map and see if we need to start IO on any
 342          * pages, fill in the partial map, etc.
 343          */
 344         index = *ppos >> PAGE_CACHE_SHIFT;
 345         nr_pages = spd.nr_pages;
 346         spd.nr_pages = 0;
 347         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
 348                 unsigned int this_len;
 349
 350                 if (!len)
 351                         break;
 352
 353                 /*
 354                  * this_len is the max we'll use from this page
 355                  */
 356                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
 357                 page = pages[page_nr];
 358
 359                 /*
 360                  * If the page isn't uptodate, we may need to start io on it
 361                  */
 362                 if (!PageUptodate(page)) {
 363                         /*
 364                          * If in nonblock mode then dont block on waiting
 365                          * for an in-flight io page
 366                          */
 367                         if (flags & SPLICE_F_NONBLOCK)
 368                                 break;
 369
 370                         lock_page(page);
 371
 372                         /*
 373                          * page was truncated, stop here. if this isn't the
 374                          * first page, we'll just complete what we already
 375                          * added
 376                          */
 377                         if (!page->mapping) {
 378                                 unlock_page(page);
 379                                 break;
 380                         }
 381                         /*
 382                          * page was already under io and is now done, great
 383                          */
 384                         if (PageUptodate(page)) {
 385                                 unlock_page(page);
 386                                 goto fill_it;
 387                         }
 388
 389                         /*
 390                          * need to read in the page
 391                          */
 392                         error = mapping->a_ops->readpage(in, page);
 393                         if (unlikely(error)) {
 394                                 /*
 395                                  * We really should re-lookup the page here,
 396                                  * but it complicates things a lot. Instead
 397                                  * lets just do what we already stored, and
 398                                  * we'll get it the next time we are called.
 399                                  */
 400                                 if (error == AOP_TRUNCATED_PAGE)
 401                                         error = 0;
 402
 403                                 break;
 404                         }
 405
 406                         /*
 407                          * i_size must be checked after ->readpage().
 408                          */
 409                         isize = i_size_read(mapping->host);
 410                         end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
 411                         if (unlikely(!isize || index > end_index))
 412                                 break;
 413
 414                         /*
 415                          * if this is the last page, see if we need to shrink
 416                          * the length and stop
 417                          */
 418                         if (end_index == index) {
 419                                 loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
 420                                 if (total_len + loff > isize)
 421                                         break;
 422                                 /*
 423                                  * force quit after adding this page
 424                                  */
 425                                 len = this_len;
 426                                 this_len = min(this_len, loff);
 427                                 loff = 0;
 428                         }
 429                 }
 430 fill_it:
 431                 partial[page_nr].offset = loff;
 432                 partial[page_nr].len = this_len;
 433                 len -= this_len;
 434                 total_len += this_len;
 435                 loff = 0;
 436                 spd.nr_pages++;
 437                 index++;
 438         }
 439
 440         /*
 441          * Release any pages at the end, if we quit early. 'i' is how far
 442          * we got, 'nr_pages' is how many pages are in the map.
 443          */
 444         while (page_nr < nr_pages)
 445                 page_cache_release(pages[page_nr++]);
 446
 447         if (spd.nr_pages)
 448                 return splice_to_pipe(pipe, &spd);
 449
 450         return error;
 451 }
 452
 453 /**
 454  * generic_file_splice_read - splice data from file to a pipe
 455  * @in:         file to splice from
 456  * @pipe:       pipe to splice to
 457  * @len:        number of bytes to splice
 458  * @flags:      splice modifier flags
 459  *
 460  * Will read pages from given file and fill them into a pipe.
 461  */
 462 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
 463                                  struct pipe_inode_info *pipe, size_t len,
 464                                  unsigned int flags)
 465 {
 466         ssize_t spliced;
 467         int ret;
 468
 469         ret = 0;
 470         spliced = 0;
 471
 472         while (len) {
 473                 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
 474
 475                 if (ret < 0)
 476                         break;
 477                 else if (!ret) {
 478                         if (spliced)
 479                                 break;
 480                         if (flags & SPLICE_F_NONBLOCK) {
 481                                 ret = -EAGAIN;
 482                                 break;
 483                         }
 484                 }
 485
 486                 *ppos += ret;
 487                 len -= ret;
 488                 spliced += ret;
 489         }
 490
 491         if (spliced)
 492                 return spliced;
 493
 494         return ret;
 495 }
 496
 497 EXPORT_SYMBOL(generic_file_splice_read);
 498
 499 /*
 500  * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
 501  * using sendpage(). Return the number of bytes sent.
 502  */
 503 static int pipe_to_sendpage(struct pipe_inode_info *info,
 504                             struct pipe_buffer *buf, struct splice_desc *sd)
 505 {
 506         struct file *file = sd->file;
 507         loff_t pos = sd->pos;
 508         int ret, more;
 509
 510         ret = buf->ops->pin(info, buf);
 511         if (!ret) {
 512                 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
 513
 514                 ret = file->f_op->sendpage(file, buf->page, buf->offset,
 515                                            sd->len, &pos, more);
 516         }
 517
 518         return ret;
 519 }
 520
 521 /*
 522  * This is a little more tricky than the file -> pipe splicing. There are
 523  * basically three cases:
 524  *
 525  *      - Destination page already exists in the address space and there
 526  *        are users of it. For that case we have no other option that
 527  *        copying the data. Tough luck.
 528  *      - Destination page already exists in the address space, but there
 529  *        are no users of it. Make sure it's uptodate, then drop it. Fall
 530  *        through to last case.
 531  *      - Destination page does not exist, we can add the pipe page to
 532  *        the page cache and avoid the copy.
 533  *
 534  * If asked to move pages to the output file (SPLICE_F_MOVE is set in
 535  * sd->flags), we attempt to migrate pages from the pipe to the output
 536  * file address space page cache. This is possible if no one else has
 537  * the pipe page referenced outside of the pipe and page cache. If
 538  * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
 539  * a new page in the output file page cache and fill/dirty that.
 540  */
 541 static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
 542                         struct splice_desc *sd)
 543 {
 544         struct file *file = sd->file;
 545         struct address_space *mapping = file->f_mapping;
 546         gfp_t gfp_mask = mapping_gfp_mask(mapping);
 547         unsigned int offset, this_len;
 548         struct page *page;
 549         pgoff_t index;
 550         int ret;
 551
 552         /*
 553          * make sure the data in this buffer is uptodate
 554          */
 555         ret = buf->ops->pin(info, buf);
 556         if (unlikely(ret))
 557                 return ret;
 558
 559         index = sd->pos >> PAGE_CACHE_SHIFT;
 560         offset = sd->pos & ~PAGE_CACHE_MASK;
 561
 562         this_len = sd->len;
 563         if (this_len + offset > PAGE_CACHE_SIZE)
 564                 this_len = PAGE_CACHE_SIZE - offset;
 565
 566         /*
 567          * Reuse buf page, if SPLICE_F_MOVE is set and we are doing a full
 568          * page.
 569          */
 570         if ((sd->flags & SPLICE_F_MOVE) && this_len == PAGE_CACHE_SIZE) {
 571                 /*
 572                  * If steal succeeds, buf->page is now pruned from the vm
 573                  * side (LRU and page cache) and we can reuse it. The page
 574                  * will also be looked on successful return.
 575                  */
 576                 if (buf->ops->steal(info, buf))
 577                         goto find_page;
 578
 579                 page = buf->page;
 580                 if (add_to_page_cache(page, mapping, index, gfp_mask)) {
 581                         unlock_page(page);
 582                         goto find_page;
 583                 }
 584
 585                 page_cache_get(page);
 586
 587                 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
 588                         lru_cache_add(page);
 589         } else {
 590 find_page:
 591                 page = find_lock_page(mapping, index);
 592                 if (!page) {
 593                         ret = -ENOMEM;
 594                         page = page_cache_alloc_cold(mapping);
 595                         if (unlikely(!page))
 596                                 goto out_nomem;
 597
 598                         /*
 599                          * This will also lock the page
 600                          */
 601                         ret = add_to_page_cache_lru(page, mapping, index,
 602                                                     gfp_mask);
 603                         if (unlikely(ret))
 604                                 goto out;
 605                 }
 606
 607                 /*
 608                  * We get here with the page locked. If the page is also
 609                  * uptodate, we don't need to do more. If it isn't, we
 610                  * may need to bring it in if we are not going to overwrite
 611                  * the full page.
 612                  */
 613                 if (!PageUptodate(page)) {
 614                         if (this_len < PAGE_CACHE_SIZE) {
 615                                 ret = mapping->a_ops->readpage(file, page);
 616                                 if (unlikely(ret))
 617                                         goto out;
 618
 619                                 lock_page(page);
 620
 621                                 if (!PageUptodate(page)) {
 622                                         /*
 623                                          * Page got invalidated, repeat.
 624                                          */
 625                                         if (!page->mapping) {
 626                                                 unlock_page(page);
 627                                                 page_cache_release(page);
 628                                                 goto find_page;
 629                                         }
 630                                         ret = -EIO;
 631                                         goto out;
 632                                 }
 633                         } else
 634                                 SetPageUptodate(page);
 635                 }
 636         }
 637
 638         ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
 639         if (ret == AOP_TRUNCATED_PAGE) {
 640                 page_cache_release(page);
 641                 goto find_page;
 642         } else if (ret)
 643                 goto out;
 644
 645         if (buf->page != page) {
 646                 /*
 647                  * Careful, ->map() uses KM_USER0!
 648                  */
 649                 char *src = buf->ops->map(info, buf, 1);
 650                 char *dst = kmap_atomic(page, KM_USER1);
 651
 652                 memcpy(dst + offset, src + buf->offset, this_len);
 653                 flush_dcache_page(page);
 654                 kunmap_atomic(dst, KM_USER1);
 655                 buf->ops->unmap(info, buf, src);
 656         }
 657
 658         ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
 659         if (!ret) {
 660                 /*
 661                  * Return the number of bytes written and mark page as
 662                  * accessed, we are now done!
 663                  */
 664                 ret = this_len;
 665                 mark_page_accessed(page);
 666                 balance_dirty_pages_ratelimited(mapping);
 667         } else if (ret == AOP_TRUNCATED_PAGE) {
 668                 page_cache_release(page);
 669                 goto find_page;
 670         }
 671 out:
 672         page_cache_release(page);
 673         unlock_page(page);
 674 out_nomem:
 675         return ret;
 676 }
 677
 678 /*
 679  * Pipe input worker. Most of this logic works like a regular pipe, the
 680  * key here is the 'actor' worker passed in that actually moves the data
 681  * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
 682  */
 683 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
 684                          loff_t *ppos, size_t len, unsigned int flags,
 685                          splice_actor *actor)
 686 {
 687         int ret, do_wakeup, err;
 688         struct splice_desc sd;
 689
 690         ret = 0;
 691         do_wakeup = 0;
 692
 693         sd.total_len = len;
 694         sd.flags = flags;
 695         sd.file = out;
 696         sd.pos = *ppos;
 697
 698         if (pipe->inode)
 699                 mutex_lock(&pipe->inode->i_mutex);
 700
 701         for (;;) {
 702                 if (pipe->nrbufs) {
 703                         struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
 704                         struct pipe_buf_operations *ops = buf->ops;
 705
 706                         sd.len = buf->len;
 707                         if (sd.len > sd.total_len)
 708                                 sd.len = sd.total_len;
 709
 710                         err = actor(pipe, buf, &sd);
 711                         if (err <= 0) {
 712                                 if (!ret && err != -ENODATA)
 713                                         ret = err;
 714
 715                                 break;
 716                         }
 717
 718                         ret += err;
 719                         buf->offset += err;
 720                         buf->len -= err;
 721
 722                         sd.len -= err;
 723                         sd.pos += err;
 724                         sd.total_len -= err;
 725                         if (sd.len)
 726                                 continue;
 727
 728                         if (!buf->len) {
 729                                 buf->ops = NULL;
 730                                 ops->release(pipe, buf);
 731                                 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
 732                                 pipe->nrbufs--;
 733                                 if (pipe->inode)
 734                                         do_wakeup = 1;
 735                         }
 736
 737                         if (!sd.total_len)
 738                                 break;
 739                 }
 740
 741                 if (pipe->nrbufs)
 742                         continue;
 743                 if (!pipe->writers)
 744                         break;
 745                 if (!pipe->waiting_writers) {
 746                         if (ret)
 747                                 break;
 748                 }
 749
 750                 if (flags & SPLICE_F_NONBLOCK) {
 751                         if (!ret)
 752                                 ret = -EAGAIN;
 753                         break;
 754                 }
 755
 756                 if (signal_pending(current)) {
 757                         if (!ret)
 758                                 ret = -ERESTARTSYS;
 759                         break;
 760                 }
 761
 762                 if (do_wakeup) {
 763                         smp_mb();
 764                         if (waitqueue_active(&pipe->wait))
 765                                 wake_up_interruptible_sync(&pipe->wait);
 766                         kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 767                         do_wakeup = 0;
 768                 }
 769
 770                 pipe_wait(pipe);
 771         }
 772
 773         if (pipe->inode)
 774                 mutex_unlock(&pipe->inode->i_mutex);
 775
 776         if (do_wakeup) {
 777                 smp_mb();
 778                 if (waitqueue_active(&pipe->wait))
 779                         wake_up_interruptible(&pipe->wait);
 780                 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 781         }
 782
 783         return ret;
 784 }
 785
 786 /**
 787  * generic_file_splice_write - splice data from a pipe to a file
 788  * @pipe:       pipe info
 789  * @out:        file to write to
 790  * @len:        number of bytes to splice
 791  * @flags:      splice modifier flags
 792  *
 793  * Will either move or copy pages (determined by @flags options) from
 794  * the given pipe inode to the given file.
 795  *
 796  */
 797 ssize_t
 798 generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
 799                           loff_t *ppos, size_t len, unsigned int flags)
 800 {
 801         struct address_space *mapping = out->f_mapping;
 802         ssize_t ret;
 803
 804         ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
 805         if (ret > 0) {
 806                 struct inode *inode = mapping->host;
 807
 808                 *ppos += ret;
 809
 810                 /*
 811                  * If file or inode is SYNC and we actually wrote some data,
 812                  * sync it.
 813                  */
 814                 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
 815                         int err;
 816
 817                         mutex_lock(&inode->i_mutex);
 818                         err = generic_osync_inode(inode, mapping,
 819                                                   OSYNC_METADATA|OSYNC_DATA);
 820                         mutex_unlock(&inode->i_mutex);
 821
 822                         if (err)
 823                                 ret = err;
 824                 }
 825         }
 826
 827         return ret;
 828 }
 829
 830 EXPORT_SYMBOL(generic_file_splice_write);
 831
 832 /**
 833  * generic_splice_sendpage - splice data from a pipe to a socket
 834  * @inode:      pipe inode
 835  * @out:        socket to write to
 836  * @len:        number of bytes to splice
 837  * @flags:      splice modifier flags
 838  *
 839  * Will send @len bytes from the pipe to a network socket. No data copying
 840  * is involved.
 841  *
 842  */
 843 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
 844                                 loff_t *ppos, size_t len, unsigned int flags)
 845 {
 846         return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
 847 }
 848
 849 EXPORT_SYMBOL(generic_splice_sendpage);
 850
 851 /*
 852  * Attempt to initiate a splice from pipe to file.
 853  */
 854 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
 855                            loff_t *ppos, size_t len, unsigned int flags)
 856 {
 857         int ret;
 858
 859         if (unlikely(!out->f_op || !out->f_op->splice_write))
 860                 return -EINVAL;
 861
 862         if (unlikely(!(out->f_mode & FMODE_WRITE)))
 863                 return -EBADF;
 864
 865         ret = rw_verify_area(WRITE, out, ppos, len);
 866         if (unlikely(ret < 0))
 867                 return ret;
 868
 869         return out->f_op->splice_write(pipe, out, ppos, len, flags);
 870 }
 871
 872 /*
 873  * Attempt to initiate a splice from a file to a pipe.
 874  */
 875 static long do_splice_to(struct file *in, loff_t *ppos,
 876                          struct pipe_inode_info *pipe, size_t len,
 877                          unsigned int flags)
 878 {
 879         loff_t isize, left;
 880         int ret;
 881
 882         if (unlikely(!in->f_op || !in->f_op->splice_read))
 883                 return -EINVAL;
 884
 885         if (unlikely(!(in->f_mode & FMODE_READ)))
 886                 return -EBADF;
 887
 888         ret = rw_verify_area(READ, in, ppos, len);
 889         if (unlikely(ret < 0))
 890                 return ret;
 891
 892         isize = i_size_read(in->f_mapping->host);
 893         if (unlikely(*ppos >= isize))
 894                 return 0;
 895
 896         left = isize - *ppos;
 897         if (unlikely(left < len))
 898                 len = left;
 899
 900         return in->f_op->splice_read(in, ppos, pipe, len, flags);
 901 }
 902
 903 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
 904                       size_t len, unsigned int flags)
 905 {
 906         struct pipe_inode_info *pipe;
 907         long ret, bytes;
 908         loff_t out_off;
 909         umode_t i_mode;
 910         int i;
 911
 912         /*
 913          * We require the input being a regular file, as we don't want to
 914          * randomly drop data for eg socket -> socket splicing. Use the
 915          * piped splicing for that!
 916          */
 917         i_mode = in->f_dentry->d_inode->i_mode;
 918         if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
 919                 return -EINVAL;
 920
 921         /*
 922          * neither in nor out is a pipe, setup an internal pipe attached to
 923          * 'out' and transfer the wanted data from 'in' to 'out' through that
 924          */
 925         pipe = current->splice_pipe;
 926         if (unlikely(!pipe)) {
 927                 pipe = alloc_pipe_info(NULL);
 928                 if (!pipe)
 929                         return -ENOMEM;
 930
 931                 /*
 932                  * We don't have an immediate reader, but we'll read the stuff
 933                  * out of the pipe right after the splice_to_pipe(). So set
 934                  * PIPE_READERS appropriately.
 935                  */
 936                 pipe->readers = 1;
 937
 938                 current->splice_pipe = pipe;
 939         }
 940
 941         /*
 942          * Do the splice.
 943          */
 944         ret = 0;
 945         bytes = 0;
 946         out_off = 0;
 947
 948         while (len) {
 949                 size_t read_len, max_read_len;
 950
 951                 /*
 952                  * Do at most PIPE_BUFFERS pages worth of transfer:
 953                  */
 954                 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
 955
 956                 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
 957                 if (unlikely(ret < 0))
 958                         goto out_release;
 959
 960                 read_len = ret;
 961
 962                 /*
 963                  * NOTE: nonblocking mode only applies to the input. We
 964                  * must not do the output in nonblocking mode as then we
 965                  * could get stuck data in the internal pipe:
 966                  */
 967                 ret = do_splice_from(pipe, out, &out_off, read_len,
 968                                      flags & ~SPLICE_F_NONBLOCK);
 969                 if (unlikely(ret < 0))
 970                         goto out_release;
 971
 972                 bytes += ret;
 973                 len -= ret;
 974
 975                 /*
 976                  * In nonblocking mode, if we got back a short read then
 977                  * that was due to either an IO error or due to the
 978                  * pagecache entry not being there. In the IO error case
 979                  * the _next_ splice attempt will produce a clean IO error
 980                  * return value (not a short read), so in both cases it's
 981                  * correct to break out of the loop here:
 982                  */
 983                 if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
 984                         break;
 985         }
 986
 987         pipe->nrbufs = pipe->curbuf = 0;
 988
 989         return bytes;
 990
 991 out_release:
 992         /*
 993          * If we did an incomplete transfer we must release
 994          * the pipe buffers in question:
 995          */
 996         for (i = 0; i < PIPE_BUFFERS; i++) {
 997                 struct pipe_buffer *buf = pipe->bufs + i;
 998
 999                 if (buf->ops) {
1000                         buf->ops->release(pipe, buf);
1001                         buf->ops = NULL;
1002                 }
1003         }
1004         pipe->nrbufs = pipe->curbuf = 0;
1005
1006         /*
1007          * If we transferred some data, return the number of bytes:
1008          */
1009         if (bytes > 0)
1010                 return bytes;
1011
1012         return ret;
1013 }
1014
1015 EXPORT_SYMBOL(do_splice_direct);
1016
1017 /*
1018  * Determine where to splice to/from.
1019  */
1020 static long do_splice(struct file *in, loff_t __user *off_in,
1021                       struct file *out, loff_t __user *off_out,
1022                       size_t len, unsigned int flags)
1023 {
1024         struct pipe_inode_info *pipe;
1025         loff_t offset, *off;
1026         long ret;
1027
1028         pipe = in->f_dentry->d_inode->i_pipe;
1029         if (pipe) {
1030                 if (off_in)
1031                         return -ESPIPE;
1032                 if (off_out) {
1033                         if (out->f_op->llseek == no_llseek)
1034                                 return -EINVAL;
1035                         if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1036                                 return -EFAULT;
1037                         off = &offset;
1038                 } else
1039                         off = &out->f_pos;
1040
1041                 ret = do_splice_from(pipe, out, off, len, flags);
1042
1043                 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1044                         ret = -EFAULT;
1045
1046                 return ret;
1047         }
1048
1049         pipe = out->f_dentry->d_inode->i_pipe;
1050         if (pipe) {
1051                 if (off_out)
1052                         return -ESPIPE;
1053                 if (off_in) {
1054                         if (in->f_op->llseek == no_llseek)
1055                                 return -EINVAL;
1056                         if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1057                                 return -EFAULT;
1058                         off = &offset;
1059                 } else
1060                         off = &in->f_pos;
1061
1062                 ret = do_splice_to(in, off, pipe, len, flags);
1063
1064                 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1065                         ret = -EFAULT;
1066
1067                 return ret;
1068         }
1069
1070         return -EINVAL;
1071 }
1072
1073 /*
1074  * Map an iov into an array of pages and offset/length tupples. With the
1075  * partial_page structure, we can map several non-contiguous ranges into
1076  * our ones pages[] map instead of splitting that operation into pieces.
1077  * Could easily be exported as a generic helper for other users, in which
1078  * case one would probably want to add a 'max_nr_pages' parameter as well.
1079  */
1080 static int get_iovec_page_array(const struct iovec __user *iov,
1081                                 unsigned int nr_vecs, struct page **pages,
1082                                 struct partial_page *partial, int aligned)
1083 {
1084         int buffers = 0, error = 0;
1085
1086         /*
1087          * It's ok to take the mmap_sem for reading, even
1088          * across a "get_user()".
1089          */
1090         down_read(&current->mm->mmap_sem);
1091
1092         while (nr_vecs) {
1093                 unsigned long off, npages;
1094                 void __user *base;
1095                 size_t len;
1096                 int i;
1097
1098                 /*
1099                  * Get user address base and length for this iovec.
1100                  */
1101                 error = get_user(base, &iov->iov_base);
1102                 if (unlikely(error))
1103                         break;
1104                 error = get_user(len, &iov->iov_len);
1105                 if (unlikely(error))
1106                         break;
1107
1108                 /*
1109                  * Sanity check this iovec. 0 read succeeds.
1110                  */
1111                 if (unlikely(!len))
1112                         break;
1113                 error = -EFAULT;
1114                 if (unlikely(!base))
1115                         break;
1116
1117                 /*
1118                  * Get this base offset and number of pages, then map
1119                  * in the user pages.
1120                  */
1121                 off = (unsigned long) base & ~PAGE_MASK;
1122
1123                 /*
1124                  * If asked for alignment, the offset must be zero and the
1125                  * length a multiple of the PAGE_SIZE.
1126                  */
1127                 error = -EINVAL;
1128                 if (aligned && (off || len & ~PAGE_MASK))
1129                         break;
1130
1131                 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1132                 if (npages > PIPE_BUFFERS - buffers)
1133                         npages = PIPE_BUFFERS - buffers;
1134
1135                 error = get_user_pages(current, current->mm,
1136                                        (unsigned long) base, npages, 0, 0,
1137                                        &pages[buffers], NULL);
1138
1139                 if (unlikely(error <= 0))
1140                         break;
1141
1142                 /*
1143                  * Fill this contiguous range into the partial page map.
1144                  */
1145                 for (i = 0; i < error; i++) {
1146                         const int plen = min_t(size_t, len, PAGE_SIZE - off);
1147
1148                         partial[buffers].offset = off;
1149                         partial[buffers].len = plen;
1150
1151                         off = 0;
1152                         len -= plen;
1153                         buffers++;
1154                 }
1155
1156                 /*
1157                  * We didn't complete this iov, stop here since it probably
1158                  * means we have to move some of this into a pipe to
1159                  * be able to continue.
1160                  */
1161                 if (len)
1162                         break;
1163
1164                 /*
1165                  * Don't continue if we mapped fewer pages than we asked for,
1166                  * or if we mapped the max number of pages that we have
1167                  * room for.
1168                  */
1169                 if (error < npages || buffers == PIPE_BUFFERS)
1170                         break;
1171
1172                 nr_vecs--;
1173                 iov++;
1174         }
1175
1176         up_read(&current->mm->mmap_sem);
1177
1178         if (buffers)
1179                 return buffers;
1180
1181         return error;
1182 }
1183
1184 /*
1185  * vmsplice splices a user address range into a pipe. It can be thought of
1186  * as splice-from-memory, where the regular splice is splice-from-file (or
1187  * to file). In both cases the output is a pipe, naturally.
1188  *
1189  * Note that vmsplice only supports splicing _from_ user memory to a pipe,
1190  * not the other way around. Splicing from user memory is a simple operation
1191  * that can be supported without any funky alignment restrictions or nasty
1192  * vm tricks. We simply map in the user memory and fill them into a pipe.
1193  * The reverse isn't quite as easy, though. There are two possible solutions
1194  * for that:
1195  *
1196  *      - memcpy() the data internally, at which point we might as well just
1197  *        do a regular read() on the buffer anyway.
1198  *      - Lots of nasty vm tricks, that are neither fast nor flexible (it
1199  *        has restriction limitations on both ends of the pipe).
1200  *
1201  * Alas, it isn't here.
1202  *
1203  */
1204 static long do_vmsplice(struct file *file, const struct iovec __user *iov,
1205                         unsigned long nr_segs, unsigned int flags)
1206 {
1207         struct pipe_inode_info *pipe = file->f_dentry->d_inode->i_pipe;
1208         struct page *pages[PIPE_BUFFERS];
1209         struct partial_page partial[PIPE_BUFFERS];
1210         struct splice_pipe_desc spd = {
1211                 .pages = pages,
1212                 .partial = partial,
1213                 .flags = flags,
1214                 .ops = &user_page_pipe_buf_ops,
1215         };
1216
1217         if (unlikely(!pipe))
1218                 return -EBADF;
1219         if (unlikely(nr_segs > UIO_MAXIOV))
1220                 return -EINVAL;
1221         else if (unlikely(!nr_segs))
1222                 return 0;
1223
1224         spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial,
1225                                             flags & SPLICE_F_GIFT);
1226         if (spd.nr_pages <= 0)
1227                 return spd.nr_pages;
1228
1229         return splice_to_pipe(pipe, &spd);
1230 }
1231
1232 asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov,
1233                              unsigned long nr_segs, unsigned int flags)
1234 {
1235         struct file *file;
1236         long error;
1237         int fput;
1238
1239         error = -EBADF;
1240         file = fget_light(fd, &fput);
1241         if (file) {
1242                 if (file->f_mode & FMODE_WRITE)
1243                         error = do_vmsplice(file, iov, nr_segs, flags);
1244
1245                 fput_light(file, fput);
1246         }
1247
1248         return error;
1249 }
1250
1251 asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
1252                            int fd_out, loff_t __user *off_out,
1253                            size_t len, unsigned int flags)
1254 {
1255         long error;
1256         struct file *in, *out;
1257         int fput_in, fput_out;
1258
1259         if (unlikely(!len))
1260                 return 0;
1261
1262         error = -EBADF;
1263         in = fget_light(fd_in, &fput_in);
1264         if (in) {
1265                 if (in->f_mode & FMODE_READ) {
1266                         out = fget_light(fd_out, &fput_out);
1267                         if (out) {
1268                                 if (out->f_mode & FMODE_WRITE)
1269                                         error = do_splice(in, off_in,
1270                                                           out, off_out,
1271                                                           len, flags);
1272                                 fput_light(out, fput_out);
1273                         }
1274                 }
1275
1276                 fput_light(in, fput_in);
1277         }
1278
1279         return error;
1280 }
1281
1282 /*
1283  * Link contents of ipipe to opipe.
1284  */
1285 static int link_pipe(struct pipe_inode_info *ipipe,
1286                      struct pipe_inode_info *opipe,
1287                      size_t len, unsigned int flags)
1288 {
1289         struct pipe_buffer *ibuf, *obuf;
1290         int ret, do_wakeup, i, ipipe_first;
1291
1292         ret = do_wakeup = ipipe_first = 0;
1293
1294         /*
1295          * Potential ABBA deadlock, work around it by ordering lock
1296          * grabbing by inode address. Otherwise two different processes
1297          * could deadlock (one doing tee from A -> B, the other from B -> A).
1298          */
1299         if (ipipe->inode < opipe->inode) {
1300                 ipipe_first = 1;
1301                 mutex_lock(&ipipe->inode->i_mutex);
1302                 mutex_lock(&opipe->inode->i_mutex);
1303         } else {
1304                 mutex_lock(&opipe->inode->i_mutex);
1305                 mutex_lock(&ipipe->inode->i_mutex);
1306         }
1307
1308         for (i = 0;; i++) {
1309                 if (!opipe->readers) {
1310                         send_sig(SIGPIPE, current, 0);
1311                         if (!ret)
1312                                 ret = -EPIPE;
1313                         break;
1314                 }
1315                 if (ipipe->nrbufs - i) {
1316                         ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1317
1318                         /*
1319                          * If we have room, fill this buffer
1320                          */
1321                         if (opipe->nrbufs < PIPE_BUFFERS) {
1322                                 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1323
1324                                 /*
1325                                  * Get a reference to this pipe buffer,
1326                                  * so we can copy the contents over.
1327                                  */
1328                                 ibuf->ops->get(ipipe, ibuf);
1329
1330                                 obuf = opipe->bufs + nbuf;
1331                                 *obuf = *ibuf;
1332
1333                                 /*
1334                                  * Don't inherit the gift flag, we need to
1335                                  * prevent multiple steals of this page.
1336                                  */
1337                                 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1338
1339                                 if (obuf->len > len)
1340                                         obuf->len = len;
1341
1342                                 opipe->nrbufs++;
1343                                 do_wakeup = 1;
1344                                 ret += obuf->len;
1345                                 len -= obuf->len;
1346
1347                                 if (!len)
1348                                         break;
1349                                 if (opipe->nrbufs < PIPE_BUFFERS)
1350                                         continue;
1351                         }
1352
1353                         /*
1354                          * We have input available, but no output room.
1355                          * If we already copied data, return that. If we
1356                          * need to drop the opipe lock, it must be ordered
1357                          * last to avoid deadlocks.
1358                          */
1359                         if ((flags & SPLICE_F_NONBLOCK) || !ipipe_first) {
1360                                 if (!ret)
1361                                         ret = -EAGAIN;
1362                                 break;
1363                         }
1364                         if (signal_pending(current)) {
1365                                 if (!ret)
1366                                         ret = -ERESTARTSYS;
1367                                 break;
1368                         }
1369                         if (do_wakeup) {
1370                                 smp_mb();
1371                                 if (waitqueue_active(&opipe->wait))
1372                                         wake_up_interruptible(&opipe->wait);
1373                                 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1374                                 do_wakeup = 0;
1375                         }
1376
1377                         opipe->waiting_writers++;
1378                         pipe_wait(opipe);
1379                         opipe->waiting_writers--;
1380                         continue;
1381                 }
1382
1383                 /*
1384                  * No input buffers, do the usual checks for available
1385                  * writers and blocking and wait if necessary
1386                  */
1387                 if (!ipipe->writers)
1388                         break;
1389                 if (!ipipe->waiting_writers) {
1390                         if (ret)
1391                                 break;
1392                 }
1393                 /*
1394                  * pipe_wait() drops the ipipe mutex. To avoid deadlocks
1395                  * with another process, we can only safely do that if
1396                  * the ipipe lock is ordered last.
1397                  */
1398                 if ((flags & SPLICE_F_NONBLOCK) || ipipe_first) {
1399                         if (!ret)
1400                                 ret = -EAGAIN;
1401                         break;
1402                 }
1403                 if (signal_pending(current)) {
1404                         if (!ret)
1405                                 ret = -ERESTARTSYS;
1406                         break;
1407                 }
1408
1409                 if (waitqueue_active(&ipipe->wait))
1410                         wake_up_interruptible_sync(&ipipe->wait);
1411                 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1412
1413                 pipe_wait(ipipe);
1414         }
1415
1416         mutex_unlock(&ipipe->inode->i_mutex);
1417         mutex_unlock(&opipe->inode->i_mutex);
1418
1419         if (do_wakeup) {
1420                 smp_mb();
1421                 if (waitqueue_active(&opipe->wait))
1422                         wake_up_interruptible(&opipe->wait);
1423                 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1424         }
1425
1426         return ret;
1427 }
1428
1429 /*
1430  * This is a tee(1) implementation that works on pipes. It doesn't copy
1431  * any data, it simply references the 'in' pages on the 'out' pipe.
1432  * The 'flags' used are the SPLICE_F_* variants, currently the only
1433  * applicable one is SPLICE_F_NONBLOCK.
1434  */
1435 static long do_tee(struct file *in, struct file *out, size_t len,
1436                    unsigned int flags)
1437 {
1438         struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1439         struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1440
1441         /*
1442          * Link ipipe to the two output pipes, consuming as we go along.
1443          */
1444         if (ipipe && opipe)
1445                 return link_pipe(ipipe, opipe, len, flags);
1446
1447         return -EINVAL;
1448 }
1449
1450 asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1451 {
1452         struct file *in;
1453         int error, fput_in;
1454
1455         if (unlikely(!len))
1456                 return 0;
1457
1458         error = -EBADF;
1459         in = fget_light(fdin, &fput_in);
1460         if (in) {
1461                 if (in->f_mode & FMODE_READ) {
1462                         int fput_out;
1463                         struct file *out = fget_light(fdout, &fput_out);
1464
1465                         if (out) {
1466                                 if (out->f_mode & FMODE_WRITE)
1467                                         error = do_tee(in, out, len, flags);
1468                                 fput_light(out, fput_out);
1469                         }
1470                 }
1471                 fput_light(in, fput_in);
1472         }
1473
1474         return error;
1475 }