SAFE public projects git trees. - safe/jmp/linux-2.6/blob - fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include "internal.h"
  18
  19 /*
  20  * check to see if a page is being written to the cache
  21  */
  22 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  23 {
  24         void *val;
  25
  26         rcu_read_lock();
  27         val = radix_tree_lookup(&cookie->stores, page->index);
  28         rcu_read_unlock();
  29
  30         return val != NULL;
  31 }
  32 EXPORT_SYMBOL(__fscache_check_page_write);
  33
  34 /*
  35  * wait for a page to finish being written to the cache
  36  */
  37 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  38 {
  39         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  40
  41         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  42 }
  43 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  44
  45 /*
  46  * decide whether a page can be released, possibly by cancelling a store to it
  47  * - we're allowed to sleep if __GFP_WAIT is flagged
  48  */
  49 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  50                                   struct page *page,
  51                                   gfp_t gfp)
  52 {
  53         struct page *xpage;
  54         void *val;
  55
  56         _enter("%p,%p,%x", cookie, page, gfp);
  57
  58         rcu_read_lock();
  59         val = radix_tree_lookup(&cookie->stores, page->index);
  60         if (!val) {
  61                 rcu_read_unlock();
  62                 fscache_stat(&fscache_n_store_vmscan_not_storing);
  63                 __fscache_uncache_page(cookie, page);
  64                 return true;
  65         }
  66
  67         /* see if the page is actually undergoing storage - if so we can't get
  68          * rid of it till the cache has finished with it */
  69         if (radix_tree_tag_get(&cookie->stores, page->index,
  70                                FSCACHE_COOKIE_STORING_TAG)) {
  71                 rcu_read_unlock();
  72                 goto page_busy;
  73         }
  74
  75         /* the page is pending storage, so we attempt to cancel the store and
  76          * discard the store request so that the page can be reclaimed */
  77         spin_lock(&cookie->stores_lock);
  78         rcu_read_unlock();
  79
  80         if (radix_tree_tag_get(&cookie->stores, page->index,
  81                                FSCACHE_COOKIE_STORING_TAG)) {
  82                 /* the page started to undergo storage whilst we were looking,
  83                  * so now we can only wait or return */
  84                 spin_unlock(&cookie->stores_lock);
  85                 goto page_busy;
  86         }
  87
  88         xpage = radix_tree_delete(&cookie->stores, page->index);
  89         spin_unlock(&cookie->stores_lock);
  90
  91         if (xpage) {
  92                 fscache_stat(&fscache_n_store_vmscan_cancelled);
  93                 fscache_stat(&fscache_n_store_radix_deletes);
  94                 ASSERTCMP(xpage, ==, page);
  95         } else {
  96                 fscache_stat(&fscache_n_store_vmscan_gone);
  97         }
  98
  99         wake_up_bit(&cookie->flags, 0);
 100         if (xpage)
 101                 page_cache_release(xpage);
 102         __fscache_uncache_page(cookie, page);
 103         return true;
 104
 105 page_busy:
 106         /* we might want to wait here, but that could deadlock the allocator as
 107          * the slow-work threads writing to the cache may all end up sleeping
 108          * on memory allocation */
 109         fscache_stat(&fscache_n_store_vmscan_busy);
 110         return false;
 111 }
 112 EXPORT_SYMBOL(__fscache_maybe_release_page);
 113
 114 /*
 115  * note that a page has finished being written to the cache
 116  */
 117 static void fscache_end_page_write(struct fscache_object *object,
 118                                    struct page *page)
 119 {
 120         struct fscache_cookie *cookie;
 121         struct page *xpage = NULL;
 122
 123         spin_lock(&object->lock);
 124         cookie = object->cookie;
 125         if (cookie) {
 126                 /* delete the page from the tree if it is now no longer
 127                  * pending */
 128                 spin_lock(&cookie->stores_lock);
 129                 radix_tree_tag_clear(&cookie->stores, page->index,
 130                                      FSCACHE_COOKIE_STORING_TAG);
 131                 if (!radix_tree_tag_get(&cookie->stores, page->index,
 132                                         FSCACHE_COOKIE_PENDING_TAG)) {
 133                         fscache_stat(&fscache_n_store_radix_deletes);
 134                         xpage = radix_tree_delete(&cookie->stores, page->index);
 135                 }
 136                 spin_unlock(&cookie->stores_lock);
 137                 wake_up_bit(&cookie->flags, 0);
 138         }
 139         spin_unlock(&object->lock);
 140         if (xpage)
 141                 page_cache_release(xpage);
 142 }
 143
 144 /*
 145  * actually apply the changed attributes to a cache object
 146  */
 147 static void fscache_attr_changed_op(struct fscache_operation *op)
 148 {
 149         struct fscache_object *object = op->object;
 150         int ret;
 151
 152         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 153
 154         fscache_stat(&fscache_n_attr_changed_calls);
 155
 156         if (fscache_object_is_active(object)) {
 157                 fscache_set_op_state(op, "CallFS");
 158                 fscache_stat(&fscache_n_cop_attr_changed);
 159                 ret = object->cache->ops->attr_changed(object);
 160                 fscache_stat_d(&fscache_n_cop_attr_changed);
 161                 fscache_set_op_state(op, "Done");
 162                 if (ret < 0)
 163                         fscache_abort_object(object);
 164         }
 165
 166         _leave("");
 167 }
 168
 169 /*
 170  * notification that the attributes on an object have changed
 171  */
 172 int __fscache_attr_changed(struct fscache_cookie *cookie)
 173 {
 174         struct fscache_operation *op;
 175         struct fscache_object *object;
 176
 177         _enter("%p", cookie);
 178
 179         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 180
 181         fscache_stat(&fscache_n_attr_changed);
 182
 183         op = kzalloc(sizeof(*op), GFP_KERNEL);
 184         if (!op) {
 185                 fscache_stat(&fscache_n_attr_changed_nomem);
 186                 _leave(" = -ENOMEM");
 187                 return -ENOMEM;
 188         }
 189
 190         fscache_operation_init(op, NULL);
 191         fscache_operation_init_slow(op, fscache_attr_changed_op);
 192         op->flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_EXCLUSIVE);
 193         fscache_set_op_name(op, "Attr");
 194
 195         spin_lock(&cookie->lock);
 196
 197         if (hlist_empty(&cookie->backing_objects))
 198                 goto nobufs;
 199         object = hlist_entry(cookie->backing_objects.first,
 200                              struct fscache_object, cookie_link);
 201
 202         if (fscache_submit_exclusive_op(object, op) < 0)
 203                 goto nobufs;
 204         spin_unlock(&cookie->lock);
 205         fscache_stat(&fscache_n_attr_changed_ok);
 206         fscache_put_operation(op);
 207         _leave(" = 0");
 208         return 0;
 209
 210 nobufs:
 211         spin_unlock(&cookie->lock);
 212         kfree(op);
 213         fscache_stat(&fscache_n_attr_changed_nobufs);
 214         _leave(" = %d", -ENOBUFS);
 215         return -ENOBUFS;
 216 }
 217 EXPORT_SYMBOL(__fscache_attr_changed);
 218
 219 /*
 220  * handle secondary execution given to a retrieval op on behalf of the
 221  * cache
 222  */
 223 static void fscache_retrieval_work(struct work_struct *work)
 224 {
 225         struct fscache_retrieval *op =
 226                 container_of(work, struct fscache_retrieval, op.fast_work);
 227         unsigned long start;
 228
 229         _enter("{OP%x}", op->op.debug_id);
 230
 231         start = jiffies;
 232         op->op.processor(&op->op);
 233         fscache_hist(fscache_ops_histogram, start);
 234         fscache_put_operation(&op->op);
 235 }
 236
 237 /*
 238  * release a retrieval op reference
 239  */
 240 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 241 {
 242         struct fscache_retrieval *op =
 243                 container_of(_op, struct fscache_retrieval, op);
 244
 245         _enter("{OP%x}", op->op.debug_id);
 246
 247         fscache_hist(fscache_retrieval_histogram, op->start_time);
 248         if (op->context)
 249                 fscache_put_context(op->op.object->cookie, op->context);
 250
 251         _leave("");
 252 }
 253
 254 /*
 255  * allocate a retrieval op
 256  */
 257 static struct fscache_retrieval *fscache_alloc_retrieval(
 258         struct address_space *mapping,
 259         fscache_rw_complete_t end_io_func,
 260         void *context)
 261 {
 262         struct fscache_retrieval *op;
 263
 264         /* allocate a retrieval operation and attempt to submit it */
 265         op = kzalloc(sizeof(*op), GFP_NOIO);
 266         if (!op) {
 267                 fscache_stat(&fscache_n_retrievals_nomem);
 268                 return NULL;
 269         }
 270
 271         fscache_operation_init(&op->op, fscache_release_retrieval_op);
 272         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
 273         op->mapping     = mapping;
 274         op->end_io_func = end_io_func;
 275         op->context     = context;
 276         op->start_time  = jiffies;
 277         INIT_WORK(&op->op.fast_work, fscache_retrieval_work);
 278         INIT_LIST_HEAD(&op->to_do);
 279         fscache_set_op_name(&op->op, "Retr");
 280         return op;
 281 }
 282
 283 /*
 284  * wait for a deferred lookup to complete
 285  */
 286 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 287 {
 288         unsigned long jif;
 289
 290         _enter("");
 291
 292         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 293                 _leave(" = 0 [imm]");
 294                 return 0;
 295         }
 296
 297         fscache_stat(&fscache_n_retrievals_wait);
 298
 299         jif = jiffies;
 300         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 301                         fscache_wait_bit_interruptible,
 302                         TASK_INTERRUPTIBLE) != 0) {
 303                 fscache_stat(&fscache_n_retrievals_intr);
 304                 _leave(" = -ERESTARTSYS");
 305                 return -ERESTARTSYS;
 306         }
 307
 308         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 309
 310         smp_rmb();
 311         fscache_hist(fscache_retrieval_delay_histogram, jif);
 312         _leave(" = 0 [dly]");
 313         return 0;
 314 }
 315
 316 /*
 317  * read a page from the cache or allocate a block in which to store it
 318  * - we return:
 319  *   -ENOMEM    - out of memory, nothing done
 320  *   -ERESTARTSYS - interrupted
 321  *   -ENOBUFS   - no backing object available in which to cache the block
 322  *   -ENODATA   - no data available in the backing object for this block
 323  *   0          - dispatched a read - it'll call end_io_func() when finished
 324  */
 325 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 326                                  struct page *page,
 327                                  fscache_rw_complete_t end_io_func,
 328                                  void *context,
 329                                  gfp_t gfp)
 330 {
 331         struct fscache_retrieval *op;
 332         struct fscache_object *object;
 333         int ret;
 334
 335         _enter("%p,%p,,,", cookie, page);
 336
 337         fscache_stat(&fscache_n_retrievals);
 338
 339         if (hlist_empty(&cookie->backing_objects))
 340                 goto nobufs;
 341
 342         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 343         ASSERTCMP(page, !=, NULL);
 344
 345         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 346                 return -ERESTARTSYS;
 347
 348         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
 349         if (!op) {
 350                 _leave(" = -ENOMEM");
 351                 return -ENOMEM;
 352         }
 353         fscache_set_op_name(&op->op, "RetrRA1");
 354
 355         spin_lock(&cookie->lock);
 356
 357         if (hlist_empty(&cookie->backing_objects))
 358                 goto nobufs_unlock;
 359         object = hlist_entry(cookie->backing_objects.first,
 360                              struct fscache_object, cookie_link);
 361
 362         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
 363
 364         atomic_inc(&object->n_reads);
 365         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 366
 367         if (fscache_submit_op(object, &op->op) < 0)
 368                 goto nobufs_unlock;
 369         spin_unlock(&cookie->lock);
 370
 371         fscache_stat(&fscache_n_retrieval_ops);
 372
 373         /* pin the netfs read context in case we need to do the actual netfs
 374          * read because we've encountered a cache read failure */
 375         fscache_get_context(object->cookie, op->context);
 376
 377         /* we wait for the operation to become active, and then process it
 378          * *here*, in this thread, and not in the thread pool */
 379         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 380                 _debug(">>> WT");
 381                 fscache_stat(&fscache_n_retrieval_op_waits);
 382                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 383                                 fscache_wait_bit_interruptible,
 384                                 TASK_INTERRUPTIBLE) < 0) {
 385                         ret = fscache_cancel_op(&op->op);
 386                         if (ret == 0) {
 387                                 ret = -ERESTARTSYS;
 388                                 goto error;
 389                         }
 390
 391                         /* it's been removed from the pending queue by another
 392                          * party, so we should get to run shortly */
 393                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 394                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 395                 }
 396                 _debug("<<< GO");
 397         }
 398
 399         /* ask the cache to honour the operation */
 400         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 401                 fscache_stat(&fscache_n_cop_allocate_page);
 402                 ret = object->cache->ops->allocate_page(op, page, gfp);
 403                 fscache_stat_d(&fscache_n_cop_allocate_page);
 404                 if (ret == 0)
 405                         ret = -ENODATA;
 406         } else {
 407                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 408                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 409                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 410         }
 411
 412 error:
 413         if (ret == -ENOMEM)
 414                 fscache_stat(&fscache_n_retrievals_nomem);
 415         else if (ret == -ERESTARTSYS)
 416                 fscache_stat(&fscache_n_retrievals_intr);
 417         else if (ret == -ENODATA)
 418                 fscache_stat(&fscache_n_retrievals_nodata);
 419         else if (ret < 0)
 420                 fscache_stat(&fscache_n_retrievals_nobufs);
 421         else
 422                 fscache_stat(&fscache_n_retrievals_ok);
 423
 424         fscache_put_retrieval(op);
 425         _leave(" = %d", ret);
 426         return ret;
 427
 428 nobufs_unlock:
 429         spin_unlock(&cookie->lock);
 430         kfree(op);
 431 nobufs:
 432         fscache_stat(&fscache_n_retrievals_nobufs);
 433         _leave(" = -ENOBUFS");
 434         return -ENOBUFS;
 435 }
 436 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 437
 438 /*
 439  * read a list of page from the cache or allocate a block in which to store
 440  * them
 441  * - we return:
 442  *   -ENOMEM    - out of memory, some pages may be being read
 443  *   -ERESTARTSYS - interrupted, some pages may be being read
 444  *   -ENOBUFS   - no backing object or space available in which to cache any
 445  *                pages not being read
 446  *   -ENODATA   - no data available in the backing object for some or all of
 447  *                the pages
 448  *   0          - dispatched a read on all pages
 449  *
 450  * end_io_func() will be called for each page read from the cache as it is
 451  * finishes being read
 452  *
 453  * any pages for which a read is dispatched will be removed from pages and
 454  * nr_pages
 455  */
 456 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 457                                   struct address_space *mapping,
 458                                   struct list_head *pages,
 459                                   unsigned *nr_pages,
 460                                   fscache_rw_complete_t end_io_func,
 461                                   void *context,
 462                                   gfp_t gfp)
 463 {
 464         struct fscache_retrieval *op;
 465         struct fscache_object *object;
 466         int ret;
 467
 468         _enter("%p,,%d,,,", cookie, *nr_pages);
 469
 470         fscache_stat(&fscache_n_retrievals);
 471
 472         if (hlist_empty(&cookie->backing_objects))
 473                 goto nobufs;
 474
 475         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 476         ASSERTCMP(*nr_pages, >, 0);
 477         ASSERT(!list_empty(pages));
 478
 479         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 480                 return -ERESTARTSYS;
 481
 482         op = fscache_alloc_retrieval(mapping, end_io_func, context);
 483         if (!op)
 484                 return -ENOMEM;
 485         fscache_set_op_name(&op->op, "RetrRAN");
 486
 487         spin_lock(&cookie->lock);
 488
 489         if (hlist_empty(&cookie->backing_objects))
 490                 goto nobufs_unlock;
 491         object = hlist_entry(cookie->backing_objects.first,
 492                              struct fscache_object, cookie_link);
 493
 494         atomic_inc(&object->n_reads);
 495         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 496
 497         if (fscache_submit_op(object, &op->op) < 0)
 498                 goto nobufs_unlock;
 499         spin_unlock(&cookie->lock);
 500
 501         fscache_stat(&fscache_n_retrieval_ops);
 502
 503         /* pin the netfs read context in case we need to do the actual netfs
 504          * read because we've encountered a cache read failure */
 505         fscache_get_context(object->cookie, op->context);
 506
 507         /* we wait for the operation to become active, and then process it
 508          * *here*, in this thread, and not in the thread pool */
 509         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 510                 _debug(">>> WT");
 511                 fscache_stat(&fscache_n_retrieval_op_waits);
 512                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 513                                 fscache_wait_bit_interruptible,
 514                                 TASK_INTERRUPTIBLE) < 0) {
 515                         ret = fscache_cancel_op(&op->op);
 516                         if (ret == 0) {
 517                                 ret = -ERESTARTSYS;
 518                                 goto error;
 519                         }
 520
 521                         /* it's been removed from the pending queue by another
 522                          * party, so we should get to run shortly */
 523                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 524                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 525                 }
 526                 _debug("<<< GO");
 527         }
 528
 529         /* ask the cache to honour the operation */
 530         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 531                 fscache_stat(&fscache_n_cop_allocate_pages);
 532                 ret = object->cache->ops->allocate_pages(
 533                         op, pages, nr_pages, gfp);
 534                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 535         } else {
 536                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 537                 ret = object->cache->ops->read_or_alloc_pages(
 538                         op, pages, nr_pages, gfp);
 539                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 540         }
 541
 542 error:
 543         if (ret == -ENOMEM)
 544                 fscache_stat(&fscache_n_retrievals_nomem);
 545         else if (ret == -ERESTARTSYS)
 546                 fscache_stat(&fscache_n_retrievals_intr);
 547         else if (ret == -ENODATA)
 548                 fscache_stat(&fscache_n_retrievals_nodata);
 549         else if (ret < 0)
 550                 fscache_stat(&fscache_n_retrievals_nobufs);
 551         else
 552                 fscache_stat(&fscache_n_retrievals_ok);
 553
 554         fscache_put_retrieval(op);
 555         _leave(" = %d", ret);
 556         return ret;
 557
 558 nobufs_unlock:
 559         spin_unlock(&cookie->lock);
 560         kfree(op);
 561 nobufs:
 562         fscache_stat(&fscache_n_retrievals_nobufs);
 563         _leave(" = -ENOBUFS");
 564         return -ENOBUFS;
 565 }
 566 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 567
 568 /*
 569  * allocate a block in the cache on which to store a page
 570  * - we return:
 571  *   -ENOMEM    - out of memory, nothing done
 572  *   -ERESTARTSYS - interrupted
 573  *   -ENOBUFS   - no backing object available in which to cache the block
 574  *   0          - block allocated
 575  */
 576 int __fscache_alloc_page(struct fscache_cookie *cookie,
 577                          struct page *page,
 578                          gfp_t gfp)
 579 {
 580         struct fscache_retrieval *op;
 581         struct fscache_object *object;
 582         int ret;
 583
 584         _enter("%p,%p,,,", cookie, page);
 585
 586         fscache_stat(&fscache_n_allocs);
 587
 588         if (hlist_empty(&cookie->backing_objects))
 589                 goto nobufs;
 590
 591         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 592         ASSERTCMP(page, !=, NULL);
 593
 594         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 595                 return -ERESTARTSYS;
 596
 597         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
 598         if (!op)
 599                 return -ENOMEM;
 600         fscache_set_op_name(&op->op, "RetrAL1");
 601
 602         spin_lock(&cookie->lock);
 603
 604         if (hlist_empty(&cookie->backing_objects))
 605                 goto nobufs_unlock;
 606         object = hlist_entry(cookie->backing_objects.first,
 607                              struct fscache_object, cookie_link);
 608
 609         if (fscache_submit_op(object, &op->op) < 0)
 610                 goto nobufs_unlock;
 611         spin_unlock(&cookie->lock);
 612
 613         fscache_stat(&fscache_n_alloc_ops);
 614
 615         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 616                 _debug(">>> WT");
 617                 fscache_stat(&fscache_n_alloc_op_waits);
 618                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 619                                 fscache_wait_bit_interruptible,
 620                                 TASK_INTERRUPTIBLE) < 0) {
 621                         ret = fscache_cancel_op(&op->op);
 622                         if (ret == 0) {
 623                                 ret = -ERESTARTSYS;
 624                                 goto error;
 625                         }
 626
 627                         /* it's been removed from the pending queue by another
 628                          * party, so we should get to run shortly */
 629                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 630                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 631                 }
 632                 _debug("<<< GO");
 633         }
 634
 635         /* ask the cache to honour the operation */
 636         fscache_stat(&fscache_n_cop_allocate_page);
 637         ret = object->cache->ops->allocate_page(op, page, gfp);
 638         fscache_stat_d(&fscache_n_cop_allocate_page);
 639
 640 error:
 641         if (ret == -ERESTARTSYS)
 642                 fscache_stat(&fscache_n_allocs_intr);
 643         else if (ret < 0)
 644                 fscache_stat(&fscache_n_allocs_nobufs);
 645         else
 646                 fscache_stat(&fscache_n_allocs_ok);
 647
 648         fscache_put_retrieval(op);
 649         _leave(" = %d", ret);
 650         return ret;
 651
 652 nobufs_unlock:
 653         spin_unlock(&cookie->lock);
 654         kfree(op);
 655 nobufs:
 656         fscache_stat(&fscache_n_allocs_nobufs);
 657         _leave(" = -ENOBUFS");
 658         return -ENOBUFS;
 659 }
 660 EXPORT_SYMBOL(__fscache_alloc_page);
 661
 662 /*
 663  * release a write op reference
 664  */
 665 static void fscache_release_write_op(struct fscache_operation *_op)
 666 {
 667         _enter("{OP%x}", _op->debug_id);
 668 }
 669
 670 /*
 671  * perform the background storage of a page into the cache
 672  */
 673 static void fscache_write_op(struct fscache_operation *_op)
 674 {
 675         struct fscache_storage *op =
 676                 container_of(_op, struct fscache_storage, op);
 677         struct fscache_object *object = op->op.object;
 678         struct fscache_cookie *cookie;
 679         struct page *page;
 680         unsigned n;
 681         void *results[1];
 682         int ret;
 683
 684         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 685
 686         fscache_set_op_state(&op->op, "GetPage");
 687
 688         spin_lock(&object->lock);
 689         cookie = object->cookie;
 690
 691         if (!fscache_object_is_active(object) || !cookie) {
 692                 spin_unlock(&object->lock);
 693                 _leave("");
 694                 return;
 695         }
 696
 697         spin_lock(&cookie->stores_lock);
 698
 699         fscache_stat(&fscache_n_store_calls);
 700
 701         /* find a page to store */
 702         page = NULL;
 703         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 704                                        FSCACHE_COOKIE_PENDING_TAG);
 705         if (n != 1)
 706                 goto superseded;
 707         page = results[0];
 708         _debug("gang %d [%lx]", n, page->index);
 709         if (page->index > op->store_limit) {
 710                 fscache_stat(&fscache_n_store_pages_over_limit);
 711                 goto superseded;
 712         }
 713
 714         if (page) {
 715                 radix_tree_tag_set(&cookie->stores, page->index,
 716                                    FSCACHE_COOKIE_STORING_TAG);
 717                 radix_tree_tag_clear(&cookie->stores, page->index,
 718                                      FSCACHE_COOKIE_PENDING_TAG);
 719         }
 720
 721         spin_unlock(&cookie->stores_lock);
 722         spin_unlock(&object->lock);
 723
 724         if (page) {
 725                 fscache_set_op_state(&op->op, "Store");
 726                 fscache_stat(&fscache_n_store_pages);
 727                 fscache_stat(&fscache_n_cop_write_page);
 728                 ret = object->cache->ops->write_page(op, page);
 729                 fscache_stat_d(&fscache_n_cop_write_page);
 730                 fscache_set_op_state(&op->op, "EndWrite");
 731                 fscache_end_page_write(object, page);
 732                 if (ret < 0) {
 733                         fscache_set_op_state(&op->op, "Abort");
 734                         fscache_abort_object(object);
 735                 } else {
 736                         fscache_enqueue_operation(&op->op);
 737                 }
 738         }
 739
 740         _leave("");
 741         return;
 742
 743 superseded:
 744         /* this writer is going away and there aren't any more things to
 745          * write */
 746         _debug("cease");
 747         spin_unlock(&cookie->stores_lock);
 748         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 749         spin_unlock(&object->lock);
 750         _leave("");
 751 }
 752
 753 /*
 754  * request a page be stored in the cache
 755  * - returns:
 756  *   -ENOMEM    - out of memory, nothing done
 757  *   -ENOBUFS   - no backing object available in which to cache the page
 758  *   0          - dispatched a write - it'll call end_io_func() when finished
 759  *
 760  * if the cookie still has a backing object at this point, that object can be
 761  * in one of a few states with respect to storage processing:
 762  *
 763  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 764  *      set)
 765  *
 766  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
 767  *          fill op)
 768  *
 769  *      (b) writes deferred till post-creation (mark page for writing and
 770  *          return immediately)
 771  *
 772  *  (2) negative lookup, object created, initial fill being made from netfs
 773  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
 774  *
 775  *      (a) fill point not yet reached this page (mark page for writing and
 776  *          return)
 777  *
 778  *      (b) fill point passed this page (queue op to store this page)
 779  *
 780  *  (3) object extant (queue op to store this page)
 781  *
 782  * any other state is invalid
 783  */
 784 int __fscache_write_page(struct fscache_cookie *cookie,
 785                          struct page *page,
 786                          gfp_t gfp)
 787 {
 788         struct fscache_storage *op;
 789         struct fscache_object *object;
 790         int ret;
 791
 792         _enter("%p,%x,", cookie, (u32) page->flags);
 793
 794         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 795         ASSERT(PageFsCache(page));
 796
 797         fscache_stat(&fscache_n_stores);
 798
 799         op = kzalloc(sizeof(*op), GFP_NOIO);
 800         if (!op)
 801                 goto nomem;
 802
 803         fscache_operation_init(&op->op, fscache_release_write_op);
 804         fscache_operation_init_slow(&op->op, fscache_write_op);
 805         op->op.flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_WAITING);
 806         fscache_set_op_name(&op->op, "Write1");
 807
 808         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
 809         if (ret < 0)
 810                 goto nomem_free;
 811
 812         ret = -ENOBUFS;
 813         spin_lock(&cookie->lock);
 814
 815         if (hlist_empty(&cookie->backing_objects))
 816                 goto nobufs;
 817         object = hlist_entry(cookie->backing_objects.first,
 818                              struct fscache_object, cookie_link);
 819         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 820                 goto nobufs;
 821
 822         /* add the page to the pending-storage radix tree on the backing
 823          * object */
 824         spin_lock(&object->lock);
 825         spin_lock(&cookie->stores_lock);
 826
 827         _debug("store limit %llx", (unsigned long long) object->store_limit);
 828
 829         ret = radix_tree_insert(&cookie->stores, page->index, page);
 830         if (ret < 0) {
 831                 if (ret == -EEXIST)
 832                         goto already_queued;
 833                 _debug("insert failed %d", ret);
 834                 goto nobufs_unlock_obj;
 835         }
 836
 837         radix_tree_tag_set(&cookie->stores, page->index,
 838                            FSCACHE_COOKIE_PENDING_TAG);
 839         page_cache_get(page);
 840
 841         /* we only want one writer at a time, but we do need to queue new
 842          * writers after exclusive ops */
 843         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 844                 goto already_pending;
 845
 846         spin_unlock(&cookie->stores_lock);
 847         spin_unlock(&object->lock);
 848
 849         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 850         op->store_limit = object->store_limit;
 851
 852         if (fscache_submit_op(object, &op->op) < 0)
 853                 goto submit_failed;
 854
 855         spin_unlock(&cookie->lock);
 856         radix_tree_preload_end();
 857         fscache_stat(&fscache_n_store_ops);
 858         fscache_stat(&fscache_n_stores_ok);
 859
 860         /* the slow work queue now carries its own ref on the object */
 861         fscache_put_operation(&op->op);
 862         _leave(" = 0");
 863         return 0;
 864
 865 already_queued:
 866         fscache_stat(&fscache_n_stores_again);
 867 already_pending:
 868         spin_unlock(&cookie->stores_lock);
 869         spin_unlock(&object->lock);
 870         spin_unlock(&cookie->lock);
 871         radix_tree_preload_end();
 872         kfree(op);
 873         fscache_stat(&fscache_n_stores_ok);
 874         _leave(" = 0");
 875         return 0;
 876
 877 submit_failed:
 878         spin_lock(&cookie->stores_lock);
 879         radix_tree_delete(&cookie->stores, page->index);
 880         spin_unlock(&cookie->stores_lock);
 881         page_cache_release(page);
 882         ret = -ENOBUFS;
 883         goto nobufs;
 884
 885 nobufs_unlock_obj:
 886         spin_unlock(&object->lock);
 887 nobufs:
 888         spin_unlock(&cookie->lock);
 889         radix_tree_preload_end();
 890         kfree(op);
 891         fscache_stat(&fscache_n_stores_nobufs);
 892         _leave(" = -ENOBUFS");
 893         return -ENOBUFS;
 894
 895 nomem_free:
 896         kfree(op);
 897 nomem:
 898         fscache_stat(&fscache_n_stores_oom);
 899         _leave(" = -ENOMEM");
 900         return -ENOMEM;
 901 }
 902 EXPORT_SYMBOL(__fscache_write_page);
 903
 904 /*
 905  * remove a page from the cache
 906  */
 907 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
 908 {
 909         struct fscache_object *object;
 910
 911         _enter(",%p", page);
 912
 913         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 914         ASSERTCMP(page, !=, NULL);
 915
 916         fscache_stat(&fscache_n_uncaches);
 917
 918         /* cache withdrawal may beat us to it */
 919         if (!PageFsCache(page))
 920                 goto done;
 921
 922         /* get the object */
 923         spin_lock(&cookie->lock);
 924
 925         if (hlist_empty(&cookie->backing_objects)) {
 926                 ClearPageFsCache(page);
 927                 goto done_unlock;
 928         }
 929
 930         object = hlist_entry(cookie->backing_objects.first,
 931                              struct fscache_object, cookie_link);
 932
 933         /* there might now be stuff on disk we could read */
 934         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 935
 936         /* only invoke the cache backend if we managed to mark the page
 937          * uncached here; this deals with synchronisation vs withdrawal */
 938         if (TestClearPageFsCache(page) &&
 939             object->cache->ops->uncache_page) {
 940                 /* the cache backend releases the cookie lock */
 941                 fscache_stat(&fscache_n_cop_uncache_page);
 942                 object->cache->ops->uncache_page(object, page);
 943                 fscache_stat_d(&fscache_n_cop_uncache_page);
 944                 goto done;
 945         }
 946
 947 done_unlock:
 948         spin_unlock(&cookie->lock);
 949 done:
 950         _leave("");
 951 }
 952 EXPORT_SYMBOL(__fscache_uncache_page);
 953
 954 /**
 955  * fscache_mark_pages_cached - Mark pages as being cached
 956  * @op: The retrieval op pages are being marked for
 957  * @pagevec: The pages to be marked
 958  *
 959  * Mark a bunch of netfs pages as being cached.  After this is called,
 960  * the netfs must call fscache_uncache_page() to remove the mark.
 961  */
 962 void fscache_mark_pages_cached(struct fscache_retrieval *op,
 963                                struct pagevec *pagevec)
 964 {
 965         struct fscache_cookie *cookie = op->op.object->cookie;
 966         unsigned long loop;
 967
 968 #ifdef CONFIG_FSCACHE_STATS
 969         atomic_add(pagevec->nr, &fscache_n_marks);
 970 #endif
 971
 972         for (loop = 0; loop < pagevec->nr; loop++) {
 973                 struct page *page = pagevec->pages[loop];
 974
 975                 _debug("- mark %p{%lx}", page, page->index);
 976                 if (TestSetPageFsCache(page)) {
 977                         static bool once_only;
 978                         if (!once_only) {
 979                                 once_only = true;
 980                                 printk(KERN_WARNING "FS-Cache:"
 981                                        " Cookie type %s marked page %lx"
 982                                        " multiple times\n",
 983                                        cookie->def->name, page->index);
 984                         }
 985                 }
 986         }
 987
 988         if (cookie->def->mark_pages_cached)
 989                 cookie->def->mark_pages_cached(cookie->netfs_data,
 990                                                op->mapping, pagevec);
 991         pagevec_reinit(pagevec);
 992 }
 993 EXPORT_SYMBOL(fscache_mark_pages_cached);