SAFE public projects git trees. - safe/jmp/linux-2.6/blob - fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include "internal.h"
  18
  19 /*
  20  * check to see if a page is being written to the cache
  21  */
  22 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  23 {
  24         void *val;
  25
  26         rcu_read_lock();
  27         val = radix_tree_lookup(&cookie->stores, page->index);
  28         rcu_read_unlock();
  29
  30         return val != NULL;
  31 }
  32 EXPORT_SYMBOL(__fscache_check_page_write);
  33
  34 /*
  35  * wait for a page to finish being written to the cache
  36  */
  37 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  38 {
  39         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  40
  41         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  42 }
  43 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  44
  45 /*
  46  * note that a page has finished being written to the cache
  47  */
  48 static void fscache_end_page_write(struct fscache_cookie *cookie, struct page *page)
  49 {
  50         struct page *xpage;
  51
  52         spin_lock(&cookie->lock);
  53         xpage = radix_tree_delete(&cookie->stores, page->index);
  54         spin_unlock(&cookie->lock);
  55         ASSERT(xpage != NULL);
  56
  57         wake_up_bit(&cookie->flags, 0);
  58 }
  59
  60 /*
  61  * actually apply the changed attributes to a cache object
  62  */
  63 static void fscache_attr_changed_op(struct fscache_operation *op)
  64 {
  65         struct fscache_object *object = op->object;
  66         int ret;
  67
  68         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
  69
  70         fscache_stat(&fscache_n_attr_changed_calls);
  71
  72         if (fscache_object_is_active(object)) {
  73                 fscache_set_op_state(op, "CallFS");
  74                 ret = object->cache->ops->attr_changed(object);
  75                 fscache_set_op_state(op, "Done");
  76                 if (ret < 0)
  77                         fscache_abort_object(object);
  78         }
  79
  80         _leave("");
  81 }
  82
  83 /*
  84  * notification that the attributes on an object have changed
  85  */
  86 int __fscache_attr_changed(struct fscache_cookie *cookie)
  87 {
  88         struct fscache_operation *op;
  89         struct fscache_object *object;
  90
  91         _enter("%p", cookie);
  92
  93         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
  94
  95         fscache_stat(&fscache_n_attr_changed);
  96
  97         op = kzalloc(sizeof(*op), GFP_KERNEL);
  98         if (!op) {
  99                 fscache_stat(&fscache_n_attr_changed_nomem);
 100                 _leave(" = -ENOMEM");
 101                 return -ENOMEM;
 102         }
 103
 104         fscache_operation_init(op, NULL);
 105         fscache_operation_init_slow(op, fscache_attr_changed_op);
 106         op->flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_EXCLUSIVE);
 107         fscache_set_op_name(op, "Attr");
 108
 109         spin_lock(&cookie->lock);
 110
 111         if (hlist_empty(&cookie->backing_objects))
 112                 goto nobufs;
 113         object = hlist_entry(cookie->backing_objects.first,
 114                              struct fscache_object, cookie_link);
 115
 116         if (fscache_submit_exclusive_op(object, op) < 0)
 117                 goto nobufs;
 118         spin_unlock(&cookie->lock);
 119         fscache_stat(&fscache_n_attr_changed_ok);
 120         fscache_put_operation(op);
 121         _leave(" = 0");
 122         return 0;
 123
 124 nobufs:
 125         spin_unlock(&cookie->lock);
 126         kfree(op);
 127         fscache_stat(&fscache_n_attr_changed_nobufs);
 128         _leave(" = %d", -ENOBUFS);
 129         return -ENOBUFS;
 130 }
 131 EXPORT_SYMBOL(__fscache_attr_changed);
 132
 133 /*
 134  * handle secondary execution given to a retrieval op on behalf of the
 135  * cache
 136  */
 137 static void fscache_retrieval_work(struct work_struct *work)
 138 {
 139         struct fscache_retrieval *op =
 140                 container_of(work, struct fscache_retrieval, op.fast_work);
 141         unsigned long start;
 142
 143         _enter("{OP%x}", op->op.debug_id);
 144
 145         start = jiffies;
 146         op->op.processor(&op->op);
 147         fscache_hist(fscache_ops_histogram, start);
 148         fscache_put_operation(&op->op);
 149 }
 150
 151 /*
 152  * release a retrieval op reference
 153  */
 154 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 155 {
 156         struct fscache_retrieval *op =
 157                 container_of(_op, struct fscache_retrieval, op);
 158
 159         _enter("{OP%x}", op->op.debug_id);
 160
 161         fscache_hist(fscache_retrieval_histogram, op->start_time);
 162         if (op->context)
 163                 fscache_put_context(op->op.object->cookie, op->context);
 164
 165         _leave("");
 166 }
 167
 168 /*
 169  * allocate a retrieval op
 170  */
 171 static struct fscache_retrieval *fscache_alloc_retrieval(
 172         struct address_space *mapping,
 173         fscache_rw_complete_t end_io_func,
 174         void *context)
 175 {
 176         struct fscache_retrieval *op;
 177
 178         /* allocate a retrieval operation and attempt to submit it */
 179         op = kzalloc(sizeof(*op), GFP_NOIO);
 180         if (!op) {
 181                 fscache_stat(&fscache_n_retrievals_nomem);
 182                 return NULL;
 183         }
 184
 185         fscache_operation_init(&op->op, fscache_release_retrieval_op);
 186         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
 187         op->mapping     = mapping;
 188         op->end_io_func = end_io_func;
 189         op->context     = context;
 190         op->start_time  = jiffies;
 191         INIT_WORK(&op->op.fast_work, fscache_retrieval_work);
 192         INIT_LIST_HEAD(&op->to_do);
 193         fscache_set_op_name(&op->op, "Retr");
 194         return op;
 195 }
 196
 197 /*
 198  * wait for a deferred lookup to complete
 199  */
 200 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 201 {
 202         unsigned long jif;
 203
 204         _enter("");
 205
 206         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 207                 _leave(" = 0 [imm]");
 208                 return 0;
 209         }
 210
 211         fscache_stat(&fscache_n_retrievals_wait);
 212
 213         jif = jiffies;
 214         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 215                         fscache_wait_bit_interruptible,
 216                         TASK_INTERRUPTIBLE) != 0) {
 217                 fscache_stat(&fscache_n_retrievals_intr);
 218                 _leave(" = -ERESTARTSYS");
 219                 return -ERESTARTSYS;
 220         }
 221
 222         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 223
 224         smp_rmb();
 225         fscache_hist(fscache_retrieval_delay_histogram, jif);
 226         _leave(" = 0 [dly]");
 227         return 0;
 228 }
 229
 230 /*
 231  * read a page from the cache or allocate a block in which to store it
 232  * - we return:
 233  *   -ENOMEM    - out of memory, nothing done
 234  *   -ERESTARTSYS - interrupted
 235  *   -ENOBUFS   - no backing object available in which to cache the block
 236  *   -ENODATA   - no data available in the backing object for this block
 237  *   0          - dispatched a read - it'll call end_io_func() when finished
 238  */
 239 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 240                                  struct page *page,
 241                                  fscache_rw_complete_t end_io_func,
 242                                  void *context,
 243                                  gfp_t gfp)
 244 {
 245         struct fscache_retrieval *op;
 246         struct fscache_object *object;
 247         int ret;
 248
 249         _enter("%p,%p,,,", cookie, page);
 250
 251         fscache_stat(&fscache_n_retrievals);
 252
 253         if (hlist_empty(&cookie->backing_objects))
 254                 goto nobufs;
 255
 256         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 257         ASSERTCMP(page, !=, NULL);
 258
 259         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 260                 return -ERESTARTSYS;
 261
 262         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
 263         if (!op) {
 264                 _leave(" = -ENOMEM");
 265                 return -ENOMEM;
 266         }
 267         fscache_set_op_name(&op->op, "RetrRA1");
 268
 269         spin_lock(&cookie->lock);
 270
 271         if (hlist_empty(&cookie->backing_objects))
 272                 goto nobufs_unlock;
 273         object = hlist_entry(cookie->backing_objects.first,
 274                              struct fscache_object, cookie_link);
 275
 276         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
 277
 278         if (fscache_submit_op(object, &op->op) < 0)
 279                 goto nobufs_unlock;
 280         spin_unlock(&cookie->lock);
 281
 282         fscache_stat(&fscache_n_retrieval_ops);
 283
 284         /* pin the netfs read context in case we need to do the actual netfs
 285          * read because we've encountered a cache read failure */
 286         fscache_get_context(object->cookie, op->context);
 287
 288         /* we wait for the operation to become active, and then process it
 289          * *here*, in this thread, and not in the thread pool */
 290         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 291                 _debug(">>> WT");
 292                 fscache_stat(&fscache_n_retrieval_op_waits);
 293                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 294                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 295                 _debug("<<< GO");
 296         }
 297
 298         /* ask the cache to honour the operation */
 299         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 300                 ret = object->cache->ops->allocate_page(op, page, gfp);
 301                 if (ret == 0)
 302                         ret = -ENODATA;
 303         } else {
 304                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 305         }
 306
 307         if (ret == -ENOMEM)
 308                 fscache_stat(&fscache_n_retrievals_nomem);
 309         else if (ret == -ERESTARTSYS)
 310                 fscache_stat(&fscache_n_retrievals_intr);
 311         else if (ret == -ENODATA)
 312                 fscache_stat(&fscache_n_retrievals_nodata);
 313         else if (ret < 0)
 314                 fscache_stat(&fscache_n_retrievals_nobufs);
 315         else
 316                 fscache_stat(&fscache_n_retrievals_ok);
 317
 318         fscache_put_retrieval(op);
 319         _leave(" = %d", ret);
 320         return ret;
 321
 322 nobufs_unlock:
 323         spin_unlock(&cookie->lock);
 324         kfree(op);
 325 nobufs:
 326         fscache_stat(&fscache_n_retrievals_nobufs);
 327         _leave(" = -ENOBUFS");
 328         return -ENOBUFS;
 329 }
 330 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 331
 332 /*
 333  * read a list of page from the cache or allocate a block in which to store
 334  * them
 335  * - we return:
 336  *   -ENOMEM    - out of memory, some pages may be being read
 337  *   -ERESTARTSYS - interrupted, some pages may be being read
 338  *   -ENOBUFS   - no backing object or space available in which to cache any
 339  *                pages not being read
 340  *   -ENODATA   - no data available in the backing object for some or all of
 341  *                the pages
 342  *   0          - dispatched a read on all pages
 343  *
 344  * end_io_func() will be called for each page read from the cache as it is
 345  * finishes being read
 346  *
 347  * any pages for which a read is dispatched will be removed from pages and
 348  * nr_pages
 349  */
 350 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 351                                   struct address_space *mapping,
 352                                   struct list_head *pages,
 353                                   unsigned *nr_pages,
 354                                   fscache_rw_complete_t end_io_func,
 355                                   void *context,
 356                                   gfp_t gfp)
 357 {
 358         fscache_pages_retrieval_func_t func;
 359         struct fscache_retrieval *op;
 360         struct fscache_object *object;
 361         int ret;
 362
 363         _enter("%p,,%d,,,", cookie, *nr_pages);
 364
 365         fscache_stat(&fscache_n_retrievals);
 366
 367         if (hlist_empty(&cookie->backing_objects))
 368                 goto nobufs;
 369
 370         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 371         ASSERTCMP(*nr_pages, >, 0);
 372         ASSERT(!list_empty(pages));
 373
 374         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 375                 return -ERESTARTSYS;
 376
 377         op = fscache_alloc_retrieval(mapping, end_io_func, context);
 378         if (!op)
 379                 return -ENOMEM;
 380         fscache_set_op_name(&op->op, "RetrRAN");
 381
 382         spin_lock(&cookie->lock);
 383
 384         if (hlist_empty(&cookie->backing_objects))
 385                 goto nobufs_unlock;
 386         object = hlist_entry(cookie->backing_objects.first,
 387                              struct fscache_object, cookie_link);
 388
 389         if (fscache_submit_op(object, &op->op) < 0)
 390                 goto nobufs_unlock;
 391         spin_unlock(&cookie->lock);
 392
 393         fscache_stat(&fscache_n_retrieval_ops);
 394
 395         /* pin the netfs read context in case we need to do the actual netfs
 396          * read because we've encountered a cache read failure */
 397         fscache_get_context(object->cookie, op->context);
 398
 399         /* we wait for the operation to become active, and then process it
 400          * *here*, in this thread, and not in the thread pool */
 401         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 402                 _debug(">>> WT");
 403                 fscache_stat(&fscache_n_retrieval_op_waits);
 404                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 405                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 406                 _debug("<<< GO");
 407         }
 408
 409         /* ask the cache to honour the operation */
 410         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags))
 411                 func = object->cache->ops->allocate_pages;
 412         else
 413                 func = object->cache->ops->read_or_alloc_pages;
 414         ret = func(op, pages, nr_pages, gfp);
 415
 416         if (ret == -ENOMEM)
 417                 fscache_stat(&fscache_n_retrievals_nomem);
 418         else if (ret == -ERESTARTSYS)
 419                 fscache_stat(&fscache_n_retrievals_intr);
 420         else if (ret == -ENODATA)
 421                 fscache_stat(&fscache_n_retrievals_nodata);
 422         else if (ret < 0)
 423                 fscache_stat(&fscache_n_retrievals_nobufs);
 424         else
 425                 fscache_stat(&fscache_n_retrievals_ok);
 426
 427         fscache_put_retrieval(op);
 428         _leave(" = %d", ret);
 429         return ret;
 430
 431 nobufs_unlock:
 432         spin_unlock(&cookie->lock);
 433         kfree(op);
 434 nobufs:
 435         fscache_stat(&fscache_n_retrievals_nobufs);
 436         _leave(" = -ENOBUFS");
 437         return -ENOBUFS;
 438 }
 439 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 440
 441 /*
 442  * allocate a block in the cache on which to store a page
 443  * - we return:
 444  *   -ENOMEM    - out of memory, nothing done
 445  *   -ERESTARTSYS - interrupted
 446  *   -ENOBUFS   - no backing object available in which to cache the block
 447  *   0          - block allocated
 448  */
 449 int __fscache_alloc_page(struct fscache_cookie *cookie,
 450                          struct page *page,
 451                          gfp_t gfp)
 452 {
 453         struct fscache_retrieval *op;
 454         struct fscache_object *object;
 455         int ret;
 456
 457         _enter("%p,%p,,,", cookie, page);
 458
 459         fscache_stat(&fscache_n_allocs);
 460
 461         if (hlist_empty(&cookie->backing_objects))
 462                 goto nobufs;
 463
 464         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 465         ASSERTCMP(page, !=, NULL);
 466
 467         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 468                 return -ERESTARTSYS;
 469
 470         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
 471         if (!op)
 472                 return -ENOMEM;
 473         fscache_set_op_name(&op->op, "RetrAL1");
 474
 475         spin_lock(&cookie->lock);
 476
 477         if (hlist_empty(&cookie->backing_objects))
 478                 goto nobufs_unlock;
 479         object = hlist_entry(cookie->backing_objects.first,
 480                              struct fscache_object, cookie_link);
 481
 482         if (fscache_submit_op(object, &op->op) < 0)
 483                 goto nobufs_unlock;
 484         spin_unlock(&cookie->lock);
 485
 486         fscache_stat(&fscache_n_alloc_ops);
 487
 488         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
 489                 _debug(">>> WT");
 490                 fscache_stat(&fscache_n_alloc_op_waits);
 491                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
 492                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 493                 _debug("<<< GO");
 494         }
 495
 496         /* ask the cache to honour the operation */
 497         ret = object->cache->ops->allocate_page(op, page, gfp);
 498
 499         if (ret < 0)
 500                 fscache_stat(&fscache_n_allocs_nobufs);
 501         else
 502                 fscache_stat(&fscache_n_allocs_ok);
 503
 504         fscache_put_retrieval(op);
 505         _leave(" = %d", ret);
 506         return ret;
 507
 508 nobufs_unlock:
 509         spin_unlock(&cookie->lock);
 510         kfree(op);
 511 nobufs:
 512         fscache_stat(&fscache_n_allocs_nobufs);
 513         _leave(" = -ENOBUFS");
 514         return -ENOBUFS;
 515 }
 516 EXPORT_SYMBOL(__fscache_alloc_page);
 517
 518 /*
 519  * release a write op reference
 520  */
 521 static void fscache_release_write_op(struct fscache_operation *_op)
 522 {
 523         _enter("{OP%x}", _op->debug_id);
 524 }
 525
 526 /*
 527  * perform the background storage of a page into the cache
 528  */
 529 static void fscache_write_op(struct fscache_operation *_op)
 530 {
 531         struct fscache_storage *op =
 532                 container_of(_op, struct fscache_storage, op);
 533         struct fscache_object *object = op->op.object;
 534         struct fscache_cookie *cookie = object->cookie;
 535         struct page *page;
 536         unsigned n;
 537         void *results[1];
 538         int ret;
 539
 540         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 541
 542         fscache_set_op_state(&op->op, "GetPage");
 543
 544         spin_lock(&cookie->lock);
 545         spin_lock(&object->lock);
 546
 547         if (!fscache_object_is_active(object)) {
 548                 spin_unlock(&object->lock);
 549                 spin_unlock(&cookie->lock);
 550                 _leave("");
 551                 return;
 552         }
 553
 554         fscache_stat(&fscache_n_store_calls);
 555
 556         /* find a page to store */
 557         page = NULL;
 558         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 559                                        FSCACHE_COOKIE_PENDING_TAG);
 560         if (n != 1)
 561                 goto superseded;
 562         page = results[0];
 563         _debug("gang %d [%lx]", n, page->index);
 564         if (page->index > op->store_limit)
 565                 goto superseded;
 566
 567         radix_tree_tag_clear(&cookie->stores, page->index,
 568                              FSCACHE_COOKIE_PENDING_TAG);
 569
 570         spin_unlock(&object->lock);
 571         spin_unlock(&cookie->lock);
 572
 573         if (page) {
 574                 fscache_set_op_state(&op->op, "Store");
 575                 ret = object->cache->ops->write_page(op, page);
 576                 fscache_set_op_state(&op->op, "EndWrite");
 577                 fscache_end_page_write(cookie, page);
 578                 page_cache_release(page);
 579                 if (ret < 0) {
 580                         fscache_set_op_state(&op->op, "Abort");
 581                         fscache_abort_object(object);
 582                 } else {
 583                         fscache_enqueue_operation(&op->op);
 584                 }
 585         }
 586
 587         _leave("");
 588         return;
 589
 590 superseded:
 591         /* this writer is going away and there aren't any more things to
 592          * write */
 593         _debug("cease");
 594         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 595         spin_unlock(&object->lock);
 596         spin_unlock(&cookie->lock);
 597         _leave("");
 598 }
 599
 600 /*
 601  * request a page be stored in the cache
 602  * - returns:
 603  *   -ENOMEM    - out of memory, nothing done
 604  *   -ENOBUFS   - no backing object available in which to cache the page
 605  *   0          - dispatched a write - it'll call end_io_func() when finished
 606  *
 607  * if the cookie still has a backing object at this point, that object can be
 608  * in one of a few states with respect to storage processing:
 609  *
 610  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 611  *      set)
 612  *
 613  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
 614  *          fill op)
 615  *
 616  *      (b) writes deferred till post-creation (mark page for writing and
 617  *          return immediately)
 618  *
 619  *  (2) negative lookup, object created, initial fill being made from netfs
 620  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
 621  *
 622  *      (a) fill point not yet reached this page (mark page for writing and
 623  *          return)
 624  *
 625  *      (b) fill point passed this page (queue op to store this page)
 626  *
 627  *  (3) object extant (queue op to store this page)
 628  *
 629  * any other state is invalid
 630  */
 631 int __fscache_write_page(struct fscache_cookie *cookie,
 632                          struct page *page,
 633                          gfp_t gfp)
 634 {
 635         struct fscache_storage *op;
 636         struct fscache_object *object;
 637         int ret;
 638
 639         _enter("%p,%x,", cookie, (u32) page->flags);
 640
 641         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 642         ASSERT(PageFsCache(page));
 643
 644         fscache_stat(&fscache_n_stores);
 645
 646         op = kzalloc(sizeof(*op), GFP_NOIO);
 647         if (!op)
 648                 goto nomem;
 649
 650         fscache_operation_init(&op->op, fscache_release_write_op);
 651         fscache_operation_init_slow(&op->op, fscache_write_op);
 652         op->op.flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_WAITING);
 653         fscache_set_op_name(&op->op, "Write1");
 654
 655         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
 656         if (ret < 0)
 657                 goto nomem_free;
 658
 659         ret = -ENOBUFS;
 660         spin_lock(&cookie->lock);
 661
 662         if (hlist_empty(&cookie->backing_objects))
 663                 goto nobufs;
 664         object = hlist_entry(cookie->backing_objects.first,
 665                              struct fscache_object, cookie_link);
 666         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 667                 goto nobufs;
 668
 669         /* add the page to the pending-storage radix tree on the backing
 670          * object */
 671         spin_lock(&object->lock);
 672
 673         _debug("store limit %llx", (unsigned long long) object->store_limit);
 674
 675         ret = radix_tree_insert(&cookie->stores, page->index, page);
 676         if (ret < 0) {
 677                 if (ret == -EEXIST)
 678                         goto already_queued;
 679                 _debug("insert failed %d", ret);
 680                 goto nobufs_unlock_obj;
 681         }
 682
 683         radix_tree_tag_set(&cookie->stores, page->index,
 684                            FSCACHE_COOKIE_PENDING_TAG);
 685         page_cache_get(page);
 686
 687         /* we only want one writer at a time, but we do need to queue new
 688          * writers after exclusive ops */
 689         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 690                 goto already_pending;
 691
 692         spin_unlock(&object->lock);
 693
 694         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 695         op->store_limit = object->store_limit;
 696
 697         if (fscache_submit_op(object, &op->op) < 0)
 698                 goto submit_failed;
 699
 700         spin_unlock(&cookie->lock);
 701         radix_tree_preload_end();
 702         fscache_stat(&fscache_n_store_ops);
 703         fscache_stat(&fscache_n_stores_ok);
 704
 705         /* the slow work queue now carries its own ref on the object */
 706         fscache_put_operation(&op->op);
 707         _leave(" = 0");
 708         return 0;
 709
 710 already_queued:
 711         fscache_stat(&fscache_n_stores_again);
 712 already_pending:
 713         spin_unlock(&object->lock);
 714         spin_unlock(&cookie->lock);
 715         radix_tree_preload_end();
 716         kfree(op);
 717         fscache_stat(&fscache_n_stores_ok);
 718         _leave(" = 0");
 719         return 0;
 720
 721 submit_failed:
 722         radix_tree_delete(&cookie->stores, page->index);
 723         page_cache_release(page);
 724         ret = -ENOBUFS;
 725         goto nobufs;
 726
 727 nobufs_unlock_obj:
 728         spin_unlock(&object->lock);
 729 nobufs:
 730         spin_unlock(&cookie->lock);
 731         radix_tree_preload_end();
 732         kfree(op);
 733         fscache_stat(&fscache_n_stores_nobufs);
 734         _leave(" = -ENOBUFS");
 735         return -ENOBUFS;
 736
 737 nomem_free:
 738         kfree(op);
 739 nomem:
 740         fscache_stat(&fscache_n_stores_oom);
 741         _leave(" = -ENOMEM");
 742         return -ENOMEM;
 743 }
 744 EXPORT_SYMBOL(__fscache_write_page);
 745
 746 /*
 747  * remove a page from the cache
 748  */
 749 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
 750 {
 751         struct fscache_object *object;
 752
 753         _enter(",%p", page);
 754
 755         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 756         ASSERTCMP(page, !=, NULL);
 757
 758         fscache_stat(&fscache_n_uncaches);
 759
 760         /* cache withdrawal may beat us to it */
 761         if (!PageFsCache(page))
 762                 goto done;
 763
 764         /* get the object */
 765         spin_lock(&cookie->lock);
 766
 767         if (hlist_empty(&cookie->backing_objects)) {
 768                 ClearPageFsCache(page);
 769                 goto done_unlock;
 770         }
 771
 772         object = hlist_entry(cookie->backing_objects.first,
 773                              struct fscache_object, cookie_link);
 774
 775         /* there might now be stuff on disk we could read */
 776         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 777
 778         /* only invoke the cache backend if we managed to mark the page
 779          * uncached here; this deals with synchronisation vs withdrawal */
 780         if (TestClearPageFsCache(page) &&
 781             object->cache->ops->uncache_page) {
 782                 /* the cache backend releases the cookie lock */
 783                 object->cache->ops->uncache_page(object, page);
 784                 goto done;
 785         }
 786
 787 done_unlock:
 788         spin_unlock(&cookie->lock);
 789 done:
 790         _leave("");
 791 }
 792 EXPORT_SYMBOL(__fscache_uncache_page);
 793
 794 /**
 795  * fscache_mark_pages_cached - Mark pages as being cached
 796  * @op: The retrieval op pages are being marked for
 797  * @pagevec: The pages to be marked
 798  *
 799  * Mark a bunch of netfs pages as being cached.  After this is called,
 800  * the netfs must call fscache_uncache_page() to remove the mark.
 801  */
 802 void fscache_mark_pages_cached(struct fscache_retrieval *op,
 803                                struct pagevec *pagevec)
 804 {
 805         struct fscache_cookie *cookie = op->op.object->cookie;
 806         unsigned long loop;
 807
 808 #ifdef CONFIG_FSCACHE_STATS
 809         atomic_add(pagevec->nr, &fscache_n_marks);
 810 #endif
 811
 812         for (loop = 0; loop < pagevec->nr; loop++) {
 813                 struct page *page = pagevec->pages[loop];
 814
 815                 _debug("- mark %p{%lx}", page, page->index);
 816                 if (TestSetPageFsCache(page)) {
 817                         static bool once_only;
 818                         if (!once_only) {
 819                                 once_only = true;
 820                                 printk(KERN_WARNING "FS-Cache:"
 821                                        " Cookie type %s marked page %lx"
 822                                        " multiple times\n",
 823                                        cookie->def->name, page->index);
 824                         }
 825                 }
 826         }
 827
 828         if (cookie->def->mark_pages_cached)
 829                 cookie->def->mark_pages_cached(cookie->netfs_data,
 830                                                op->mapping, pagevec);
 831         pagevec_reinit(pagevec);
 832 }
 833 EXPORT_SYMBOL(fscache_mark_pages_cached);