X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=drivers%2Fmd%2Fdm-snap.c;h=3a3ba46e6d4b68fc9f70b87b7ad322a8ca2c9846;hb=76bae570899be34317510d8006d490572152bdfb;hp=475514bda9d0f7c87c131e7f361e2b35a8090030;hpb=b4b610f684d13bf8691feeae5d4d7a8bd1f1033e;p=safe%2Fjmp%2Flinux-2.6 diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 475514b..3a3ba46 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -7,9 +7,8 @@ */ #include -#include -#include #include +#include #include #include #include @@ -18,10 +17,13 @@ #include #include #include +#include +#include +#include -#include "dm-snap.h" -#include "dm-bio-list.h" -#include "kcopyd.h" +#include "dm-exception-store.h" + +#define DM_MSG_PREFIX "snapshots" /* * The percentage increment we will wake up users at @@ -34,12 +36,87 @@ #define SNAPSHOT_COPY_PRIORITY 2 /* - * Each snapshot reserves this many pages for io + * Reserve 1MB for each snapshot initially (with minimum of 1 page). + */ +#define SNAPSHOT_PAGES (((1UL << 20) >> PAGE_SHIFT) ? : 1) + +/* + * The size of the mempool used to track chunks in use. */ -#define SNAPSHOT_PAGES 256 +#define MIN_IOS 256 + +#define DM_TRACKED_CHUNK_HASH_SIZE 16 +#define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ + (DM_TRACKED_CHUNK_HASH_SIZE - 1)) + +struct exception_table { + uint32_t hash_mask; + unsigned hash_shift; + struct list_head *table; +}; + +struct dm_snapshot { + struct rw_semaphore lock; + + struct dm_dev *origin; + + /* List of snapshots per Origin */ + struct list_head list; -struct pending_exception { - struct exception e; + /* You can't use a snapshot if this is 0 (e.g. if full) */ + int valid; + + /* Origin writes don't trigger exceptions until this is set */ + int active; + + mempool_t *pending_pool; + + atomic_t pending_exceptions_count; + + struct exception_table pending; + struct exception_table complete; + + /* + * pe_lock protects all pending_exception operations and access + * as well as the snapshot_bios list. + */ + spinlock_t pe_lock; + + /* The on disk metadata handler */ + struct dm_exception_store *store; + + struct dm_kcopyd_client *kcopyd_client; + + /* Queue of snapshot writes for ksnapd to flush */ + struct bio_list queued_bios; + struct work_struct queued_bios_work; + + /* Chunks with outstanding reads */ + mempool_t *tracked_chunk_pool; + spinlock_t tracked_chunk_lock; + struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE]; +}; + +static struct workqueue_struct *ksnapd; +static void flush_queued_bios(struct work_struct *work); + +static sector_t chunk_to_sector(struct dm_exception_store *store, + chunk_t chunk) +{ + return chunk << store->chunk_shift; +} + +static int bdev_equal(struct block_device *lhs, struct block_device *rhs) +{ + /* + * There is only ever one instance of a particular block + * device so we can compare pointers safely. + */ + return lhs == rhs; +} + +struct dm_snap_pending_exception { + struct dm_snap_exception e; /* * Origin buffers waiting for this to complete are held @@ -55,11 +132,11 @@ struct pending_exception { /* * The primary pending_exception is the one that holds - * the sibling_count and the list of origin_bios for a + * the ref_count and the list of origin_bios for a * group of pending_exceptions. It is always last to get freed. * These fields get set up when writing to the origin. */ - struct pending_exception *primary_pe; + struct dm_snap_pending_exception *primary_pe; /* * Number of pending_exceptions processing this chunk. @@ -68,7 +145,7 @@ struct pending_exception { * the sibling concerned and not pe->primary_pe->snap->lock unless * they are the same. */ - atomic_t sibling_count; + atomic_t ref_count; /* Pointer back to snapshot context */ struct dm_snapshot *snap; @@ -84,9 +161,65 @@ struct pending_exception { * Hash table mapping origin volumes to lists of snapshots and * a lock to protect it */ -static kmem_cache_t *exception_cache; -static kmem_cache_t *pending_cache; -static mempool_t *pending_pool; +static struct kmem_cache *exception_cache; +static struct kmem_cache *pending_cache; + +struct dm_snap_tracked_chunk { + struct hlist_node node; + chunk_t chunk; +}; + +static struct kmem_cache *tracked_chunk_cache; + +static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s, + chunk_t chunk) +{ + struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool, + GFP_NOIO); + unsigned long flags; + + c->chunk = chunk; + + spin_lock_irqsave(&s->tracked_chunk_lock, flags); + hlist_add_head(&c->node, + &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); + spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); + + return c; +} + +static void stop_tracking_chunk(struct dm_snapshot *s, + struct dm_snap_tracked_chunk *c) +{ + unsigned long flags; + + spin_lock_irqsave(&s->tracked_chunk_lock, flags); + hlist_del(&c->node); + spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); + + mempool_free(c, s->tracked_chunk_pool); +} + +static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk) +{ + struct dm_snap_tracked_chunk *c; + struct hlist_node *hn; + int found = 0; + + spin_lock_irq(&s->tracked_chunk_lock); + + hlist_for_each_entry(c, hn, + &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { + if (c->chunk == chunk) { + found = 1; + break; + } + } + + spin_unlock_irq(&s->tracked_chunk_lock); + + return found; +} /* * One of these per registered origin, held in the snapshot_origins hash @@ -117,7 +250,7 @@ static int init_origin_hash(void) _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), GFP_KERNEL); if (!_origins) { - DMERR("Device mapper: Snapshot: unable to allocate memory"); + DMERR("unable to allocate memory"); return -ENOMEM; } @@ -133,7 +266,7 @@ static void exit_origin_hash(void) kfree(_origins); } -static inline unsigned int origin_hash(struct block_device *bdev) +static unsigned origin_hash(struct block_device *bdev) { return bdev->bd_dev & ORIGIN_MASK; } @@ -163,19 +296,22 @@ static void __insert_origin(struct origin *o) */ static int register_snapshot(struct dm_snapshot *snap) { - struct origin *o; + struct dm_snapshot *l; + struct origin *o, *new_o; struct block_device *bdev = snap->origin->bdev; + new_o = kmalloc(sizeof(*new_o), GFP_KERNEL); + if (!new_o) + return -ENOMEM; + down_write(&_origins_lock); o = __lookup_origin(bdev); - if (!o) { + if (o) + kfree(new_o); + else { /* New origin */ - o = kmalloc(sizeof(*o), GFP_KERNEL); - if (!o) { - up_write(&_origins_lock); - return -ENOMEM; - } + o = new_o; /* Initialise the struct */ INIT_LIST_HEAD(&o->snapshots); @@ -184,7 +320,11 @@ static int register_snapshot(struct dm_snapshot *snap) __insert_origin(o); } - list_add_tail(&snap->list, &o->snapshots); + /* Sort the list according to chunk size, largest-first smallest-last */ + list_for_each_entry(l, &o->snapshots, list) + if (l->store->chunk_size < snap->store->chunk_size) + break; + list_add_tail(&snap->list, &l->list); up_write(&_origins_lock); return 0; @@ -208,11 +348,15 @@ static void unregister_snapshot(struct dm_snapshot *s) /* * Implementation of the exception hash tables. + * The lowest hash_shift bits of the chunk number are ignored, allowing + * some consecutive chunks to be grouped together. */ -static int init_exception_table(struct exception_table *et, uint32_t size) +static int init_exception_table(struct exception_table *et, uint32_t size, + unsigned hash_shift) { unsigned int i; + et->hash_shift = hash_shift; et->hash_mask = size - 1; et->table = dm_vcalloc(size, sizeof(struct list_head)); if (!et->table) @@ -224,10 +368,10 @@ static int init_exception_table(struct exception_table *et, uint32_t size) return 0; } -static void exit_exception_table(struct exception_table *et, kmem_cache_t *mem) +static void exit_exception_table(struct exception_table *et, struct kmem_cache *mem) { struct list_head *slot; - struct exception *ex, *next; + struct dm_snap_exception *ex, *next; int i, size; size = et->hash_mask + 1; @@ -241,18 +385,19 @@ static void exit_exception_table(struct exception_table *et, kmem_cache_t *mem) vfree(et->table); } -static inline uint32_t exception_hash(struct exception_table *et, chunk_t chunk) +static uint32_t exception_hash(struct exception_table *et, chunk_t chunk) { - return chunk & et->hash_mask; + return (chunk >> et->hash_shift) & et->hash_mask; } -static void insert_exception(struct exception_table *eh, struct exception *e) +static void insert_exception(struct exception_table *eh, + struct dm_snap_exception *e) { struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)]; list_add(&e->hash_list, l); } -static inline void remove_exception(struct exception *e) +static void remove_exception(struct dm_snap_exception *e) { list_del(&e->hash_list); } @@ -261,23 +406,24 @@ static inline void remove_exception(struct exception *e) * Return the exception data for a sector, or NULL if not * remapped. */ -static struct exception *lookup_exception(struct exception_table *et, - chunk_t chunk) +static struct dm_snap_exception *lookup_exception(struct exception_table *et, + chunk_t chunk) { struct list_head *slot; - struct exception *e; + struct dm_snap_exception *e; slot = &et->table[exception_hash(et, chunk)]; list_for_each_entry (e, slot, hash_list) - if (e->old_chunk == chunk) + if (chunk >= e->old_chunk && + chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) return e; return NULL; } -static inline struct exception *alloc_exception(void) +static struct dm_snap_exception *alloc_exception(void) { - struct exception *e; + struct dm_snap_exception *e; e = kmem_cache_alloc(exception_cache, GFP_NOIO); if (!e) @@ -286,32 +432,94 @@ static inline struct exception *alloc_exception(void) return e; } -static inline void free_exception(struct exception *e) +static void free_exception(struct dm_snap_exception *e) { kmem_cache_free(exception_cache, e); } -static inline struct pending_exception *alloc_pending_exception(void) +static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s) { - return mempool_alloc(pending_pool, GFP_NOIO); + struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool, + GFP_NOIO); + + atomic_inc(&s->pending_exceptions_count); + pe->snap = s; + + return pe; } -static inline void free_pending_exception(struct pending_exception *pe) +static void free_pending_exception(struct dm_snap_pending_exception *pe) { - mempool_free(pe, pending_pool); + struct dm_snapshot *s = pe->snap; + + mempool_free(pe, s->pending_pool); + smp_mb__before_atomic_dec(); + atomic_dec(&s->pending_exceptions_count); } -int dm_add_exception(struct dm_snapshot *s, chunk_t old, chunk_t new) +static void insert_completed_exception(struct dm_snapshot *s, + struct dm_snap_exception *new_e) { - struct exception *e; + struct exception_table *eh = &s->complete; + struct list_head *l; + struct dm_snap_exception *e = NULL; + + l = &eh->table[exception_hash(eh, new_e->old_chunk)]; + + /* Add immediately if this table doesn't support consecutive chunks */ + if (!eh->hash_shift) + goto out; + + /* List is ordered by old_chunk */ + list_for_each_entry_reverse(e, l, hash_list) { + /* Insert after an existing chunk? */ + if (new_e->old_chunk == (e->old_chunk + + dm_consecutive_chunk_count(e) + 1) && + new_e->new_chunk == (dm_chunk_number(e->new_chunk) + + dm_consecutive_chunk_count(e) + 1)) { + dm_consecutive_chunk_count_inc(e); + free_exception(new_e); + return; + } + + /* Insert before an existing chunk? */ + if (new_e->old_chunk == (e->old_chunk - 1) && + new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) { + dm_consecutive_chunk_count_inc(e); + e->old_chunk--; + e->new_chunk--; + free_exception(new_e); + return; + } + + if (new_e->old_chunk > e->old_chunk) + break; + } + +out: + list_add(&new_e->hash_list, e ? &e->hash_list : l); +} + +/* + * Callback used by the exception stores to load exceptions when + * initialising. + */ +static int dm_add_exception(void *context, chunk_t old, chunk_t new) +{ + struct dm_snapshot *s = context; + struct dm_snap_exception *e; e = alloc_exception(); if (!e) return -ENOMEM; e->old_chunk = old; + + /* Consecutive_count is implicitly initialised to zero */ e->new_chunk = new; - insert_exception(&s->complete, e); + + insert_completed_exception(s, e); + return 0; } @@ -328,16 +536,6 @@ static int calc_max_buckets(void) } /* - * Rounds a number down to a power of 2. - */ -static inline uint32_t round_down(uint32_t n) -{ - while (n & (n - 1)) - n &= (n - 1); - return n; -} - -/* * Allocate room for a suitable hash table. */ static int init_hash_tables(struct dm_snapshot *s) @@ -348,16 +546,16 @@ static int init_hash_tables(struct dm_snapshot *s) * Calculate based on the size of the original volume or * the COW volume... */ - cow_dev_size = get_dev_size(s->cow->bdev); + cow_dev_size = get_dev_size(s->store->cow->bdev); origin_dev_size = get_dev_size(s->origin->bdev); max_buckets = calc_max_buckets(); - hash_size = min(origin_dev_size, cow_dev_size) >> s->chunk_shift; + hash_size = min(origin_dev_size, cow_dev_size) >> s->store->chunk_shift; hash_size = min(hash_size, max_buckets); - /* Round it down to a power of 2 */ - hash_size = round_down(hash_size); - if (init_exception_table(&s->complete, hash_size)) + hash_size = rounddown_pow_of_two(hash_size); + if (init_exception_table(&s->complete, hash_size, + DM_CHUNK_CONSECUTIVE_BITS)) return -ENOMEM; /* @@ -368,7 +566,7 @@ static int init_hash_tables(struct dm_snapshot *s) if (hash_size < 64) hash_size = 64; - if (init_exception_table(&s->pending, hash_size)) { + if (init_exception_table(&s->pending, hash_size, 0)) { exit_exception_table(&s->complete, exception_cache); return -ENOMEM; } @@ -377,197 +575,194 @@ static int init_hash_tables(struct dm_snapshot *s) } /* - * Round a number up to the nearest 'size' boundary. size must - * be a power of 2. - */ -static inline ulong round_up(ulong n, ulong size) -{ - size--; - return (n + size) & ~size; -} - -static void read_snapshot_metadata(struct dm_snapshot *s) -{ - if (s->store.read_metadata(&s->store)) { - down_write(&s->lock); - s->valid = 0; - up_write(&s->lock); - } -} - -/* * Construct a snapshot mapping:

*/ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) { struct dm_snapshot *s; - unsigned long chunk_size; + int i; int r = -EINVAL; - char persistent; char *origin_path; - char *cow_path; - char *value; - int blocksize; + struct dm_exception_store *store; + unsigned args_used; - if (argc < 4) { - ti->error = "dm-snapshot: requires exactly 4 arguments"; + if (argc != 4) { + ti->error = "requires exactly 4 arguments"; r = -EINVAL; - goto bad1; + goto bad_args; } origin_path = argv[0]; - cow_path = argv[1]; - persistent = toupper(*argv[2]); + argv++; + argc--; - if (persistent != 'P' && persistent != 'N') { - ti->error = "Persistent flag is not P or N"; + r = dm_exception_store_create(ti, argc, argv, &args_used, &store); + if (r) { + ti->error = "Couldn't create exception store"; r = -EINVAL; - goto bad1; + goto bad_args; } - chunk_size = simple_strtoul(argv[3], &value, 10); - if (chunk_size == 0 || value == NULL) { - ti->error = "Invalid chunk size"; - r = -EINVAL; - goto bad1; - } + argv += args_used; + argc -= args_used; s = kmalloc(sizeof(*s), GFP_KERNEL); - if (s == NULL) { + if (!s) { ti->error = "Cannot allocate snapshot context private " "structure"; r = -ENOMEM; - goto bad1; + goto bad_snap; } r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin); if (r) { ti->error = "Cannot get origin device"; - goto bad2; + goto bad_origin; } - r = dm_get_device(ti, cow_path, 0, 0, - FMODE_READ | FMODE_WRITE, &s->cow); - if (r) { - dm_put_device(ti, s->origin); - ti->error = "Cannot get COW device"; - goto bad2; - } - - /* - * Chunk size must be multiple of page size. Silently - * round up if it's not. - */ - chunk_size = round_up(chunk_size, PAGE_SIZE >> 9); - - /* Validate the chunk size against the device block size */ - blocksize = s->cow->bdev->bd_disk->queue->hardsect_size; - if (chunk_size % (blocksize >> 9)) { - ti->error = "Chunk size is not a multiple of device blocksize"; - r = -EINVAL; - goto bad3; - } - - /* Check chunk_size is a power of 2 */ - if (chunk_size & (chunk_size - 1)) { - ti->error = "Chunk size is not a power of 2"; - r = -EINVAL; - goto bad3; - } - - s->chunk_size = chunk_size; - s->chunk_mask = chunk_size - 1; - s->type = persistent; - s->chunk_shift = ffs(chunk_size) - 1; - + s->store = store; s->valid = 1; s->active = 0; - s->last_percent = 0; + atomic_set(&s->pending_exceptions_count, 0); init_rwsem(&s->lock); - s->table = ti->table; + spin_lock_init(&s->pe_lock); /* Allocate hash table for COW data */ if (init_hash_tables(s)) { ti->error = "Unable to allocate hash table space"; r = -ENOMEM; - goto bad3; + goto bad_hash_tables; } - /* - * Check the persistent flag - done here because we need the iobuf - * to check the LV header - */ - s->store.snap = s; - - if (persistent == 'P') - r = dm_create_persistent(&s->store, chunk_size); - else - r = dm_create_transient(&s->store, s, blocksize); - + r = dm_kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client); if (r) { - ti->error = "Couldn't create exception store"; - r = -EINVAL; - goto bad4; + ti->error = "Could not create kcopyd client"; + goto bad_kcopyd; } - r = kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client); - if (r) { - ti->error = "Could not create kcopyd client"; - goto bad5; + s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache); + if (!s->pending_pool) { + ti->error = "Could not allocate mempool for pending exceptions"; + goto bad_pending_pool; } + s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS, + tracked_chunk_cache); + if (!s->tracked_chunk_pool) { + ti->error = "Could not allocate tracked_chunk mempool for " + "tracking reads"; + goto bad_tracked_chunk_pool; + } + + for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) + INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]); + + spin_lock_init(&s->tracked_chunk_lock); + /* Metadata must only be loaded into one table at once */ - read_snapshot_metadata(s); + r = s->store->type->read_metadata(s->store, dm_add_exception, + (void *)s); + if (r < 0) { + ti->error = "Failed to read snapshot metadata"; + goto bad_load_and_register; + } else if (r > 0) { + s->valid = 0; + DMWARN("Snapshot is marked invalid."); + } + + bio_list_init(&s->queued_bios); + INIT_WORK(&s->queued_bios_work, flush_queued_bios); + + if (!s->store->chunk_size) { + ti->error = "Chunk size not set"; + goto bad_load_and_register; + } /* Add snapshot to the list of snapshots for this origin */ /* Exceptions aren't triggered till snapshot_resume() is called */ if (register_snapshot(s)) { r = -EINVAL; ti->error = "Cannot register snapshot origin"; - goto bad6; + goto bad_load_and_register; } ti->private = s; - ti->split_io = chunk_size; + ti->split_io = s->store->chunk_size; + ti->num_flush_requests = 1; return 0; - bad6: - kcopyd_client_destroy(s->kcopyd_client); +bad_load_and_register: + mempool_destroy(s->tracked_chunk_pool); - bad5: - s->store.destroy(&s->store); +bad_tracked_chunk_pool: + mempool_destroy(s->pending_pool); - bad4: +bad_pending_pool: + dm_kcopyd_client_destroy(s->kcopyd_client); + +bad_kcopyd: exit_exception_table(&s->pending, pending_cache); exit_exception_table(&s->complete, exception_cache); - bad3: - dm_put_device(ti, s->cow); +bad_hash_tables: dm_put_device(ti, s->origin); - bad2: +bad_origin: kfree(s); - bad1: +bad_snap: + dm_exception_store_destroy(store); + +bad_args: return r; } +static void __free_exceptions(struct dm_snapshot *s) +{ + dm_kcopyd_client_destroy(s->kcopyd_client); + s->kcopyd_client = NULL; + + exit_exception_table(&s->pending, pending_cache); + exit_exception_table(&s->complete, exception_cache); +} + static void snapshot_dtr(struct dm_target *ti) { - struct dm_snapshot *s = (struct dm_snapshot *) ti->private; +#ifdef CONFIG_DM_DEBUG + int i; +#endif + struct dm_snapshot *s = ti->private; + flush_workqueue(ksnapd); + + /* Prevent further origin writes from using this snapshot. */ + /* After this returns there can be no new kcopyd jobs. */ unregister_snapshot(s); - exit_exception_table(&s->pending, pending_cache); - exit_exception_table(&s->complete, exception_cache); + while (atomic_read(&s->pending_exceptions_count)) + msleep(1); + /* + * Ensure instructions in mempool_destroy aren't reordered + * before atomic_read. + */ + smp_mb(); + +#ifdef CONFIG_DM_DEBUG + for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) + BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i])); +#endif + + mempool_destroy(s->tracked_chunk_pool); - /* Deallocate memory used */ - s->store.destroy(&s->store); + __free_exceptions(s); + + mempool_destroy(s->pending_pool); dm_put_device(ti, s->origin); - dm_put_device(ti, s->cow); - kcopyd_client_destroy(s->kcopyd_client); + + dm_exception_store_destroy(s->store); + kfree(s); } @@ -586,6 +781,20 @@ static void flush_bios(struct bio *bio) } } +static void flush_queued_bios(struct work_struct *work) +{ + struct dm_snapshot *s = + container_of(work, struct dm_snapshot, queued_bios_work); + struct bio *queued_bios; + unsigned long flags; + + spin_lock_irqsave(&s->pe_lock, flags); + queued_bios = bio_list_get(&s->queued_bios); + spin_unlock_irqrestore(&s->pe_lock, flags); + + flush_bios(queued_bios); +} + /* * Error a list of buffers. */ @@ -596,100 +805,127 @@ static void error_bios(struct bio *bio) while (bio) { n = bio->bi_next; bio->bi_next = NULL; - bio_io_error(bio, bio->bi_size); + bio_io_error(bio); bio = n; } } -static struct bio *__flush_bios(struct pending_exception *pe) +static void __invalidate_snapshot(struct dm_snapshot *s, int err) +{ + if (!s->valid) + return; + + if (err == -EIO) + DMERR("Invalidating snapshot: Error reading/writing."); + else if (err == -ENOMEM) + DMERR("Invalidating snapshot: Unable to allocate exception."); + + if (s->store->type->drop_snapshot) + s->store->type->drop_snapshot(s->store); + + s->valid = 0; + + dm_table_event(s->store->ti->table); +} + +static void get_pending_exception(struct dm_snap_pending_exception *pe) { + atomic_inc(&pe->ref_count); +} + +static struct bio *put_pending_exception(struct dm_snap_pending_exception *pe) +{ + struct dm_snap_pending_exception *primary_pe; + struct bio *origin_bios = NULL; + + primary_pe = pe->primary_pe; + /* * If this pe is involved in a write to the origin and * it is the last sibling to complete then release * the bios for the original write to the origin. */ + if (primary_pe && + atomic_dec_and_test(&primary_pe->ref_count)) { + origin_bios = bio_list_get(&primary_pe->origin_bios); + free_pending_exception(primary_pe); + } - if (pe->primary_pe && - atomic_dec_and_test(&pe->primary_pe->sibling_count)) - return bio_list_get(&pe->primary_pe->origin_bios); + /* + * Free the pe if it's not linked to an origin write or if + * it's not itself a primary pe. + */ + if (!primary_pe || primary_pe != pe) + free_pending_exception(pe); - return NULL; + return origin_bios; } -static void pending_complete(struct pending_exception *pe, int success) +static void pending_complete(struct dm_snap_pending_exception *pe, int success) { - struct exception *e; - struct pending_exception *primary_pe; + struct dm_snap_exception *e; struct dm_snapshot *s = pe->snap; - struct bio *flush = NULL; + struct bio *origin_bios = NULL; + struct bio *snapshot_bios = NULL; + int error = 0; - if (success) { - e = alloc_exception(); - if (!e) { - DMWARN("Unable to allocate exception."); - down_write(&s->lock); - s->store.drop_snapshot(&s->store); - s->valid = 0; - flush = __flush_bios(pe); - up_write(&s->lock); - - error_bios(bio_list_get(&pe->snapshot_bios)); - goto out; - } - *e = pe->e; - - /* - * Add a proper exception, and remove the - * in-flight exception from the list. - */ - down_write(&s->lock); - insert_exception(&s->complete, e); - remove_exception(&pe->e); - flush = __flush_bios(pe); - - /* Submit any pending write bios */ - up_write(&s->lock); - - flush_bios(bio_list_get(&pe->snapshot_bios)); - } else { + if (!success) { /* Read/write error - snapshot is unusable */ down_write(&s->lock); - if (s->valid) - DMERR("Error reading/writing snapshot"); - s->store.drop_snapshot(&s->store); - s->valid = 0; - remove_exception(&pe->e); - flush = __flush_bios(pe); - up_write(&s->lock); - - error_bios(bio_list_get(&pe->snapshot_bios)); + __invalidate_snapshot(s, -EIO); + error = 1; + goto out; + } - dm_table_event(s->table); + e = alloc_exception(); + if (!e) { + down_write(&s->lock); + __invalidate_snapshot(s, -ENOMEM); + error = 1; + goto out; } + *e = pe->e; - out: - primary_pe = pe->primary_pe; + down_write(&s->lock); + if (!s->valid) { + free_exception(e); + error = 1; + goto out; + } /* - * Free the pe if it's not linked to an origin write or if - * it's not itself a primary pe. + * Check for conflicting reads. This is extremely improbable, + * so msleep(1) is sufficient and there is no need for a wait queue. */ - if (!primary_pe || primary_pe != pe) - free_pending_exception(pe); + while (__chunk_is_tracked(s, pe->e.old_chunk)) + msleep(1); /* - * Free the primary pe if nothing references it. + * Add a proper exception, and remove the + * in-flight exception from the list. */ - if (primary_pe && !atomic_read(&primary_pe->sibling_count)) - free_pending_exception(primary_pe); + insert_completed_exception(s, e); + + out: + remove_exception(&pe->e); + snapshot_bios = bio_list_get(&pe->snapshot_bios); + origin_bios = put_pending_exception(pe); - if (flush) - flush_bios(flush); + up_write(&s->lock); + + /* Submit any pending write bios */ + if (error) + error_bios(snapshot_bios); + else + flush_bios(snapshot_bios); + + flush_bios(origin_bios); } static void commit_callback(void *context, int success) { - struct pending_exception *pe = (struct pending_exception *) context; + struct dm_snap_pending_exception *pe = context; + pending_complete(pe, success); } @@ -697,9 +933,9 @@ static void commit_callback(void *context, int success) * Called when the copy I/O has finished. kcopyd actually runs * this code so don't block. */ -static void copy_callback(int read_err, unsigned int write_err, void *context) +static void copy_callback(int read_err, unsigned long write_err, void *context) { - struct pending_exception *pe = (struct pending_exception *) context; + struct dm_snap_pending_exception *pe = context; struct dm_snapshot *s = pe->snap; if (read_err || write_err) @@ -707,35 +943,46 @@ static void copy_callback(int read_err, unsigned int write_err, void *context) else /* Update the metadata if we are persistent */ - s->store.commit_exception(&s->store, &pe->e, commit_callback, - pe); + s->store->type->commit_exception(s->store, &pe->e, + commit_callback, pe); } /* * Dispatches the copy operation to kcopyd. */ -static void start_copy(struct pending_exception *pe) +static void start_copy(struct dm_snap_pending_exception *pe) { struct dm_snapshot *s = pe->snap; - struct io_region src, dest; + struct dm_io_region src, dest; struct block_device *bdev = s->origin->bdev; sector_t dev_size; dev_size = get_dev_size(bdev); src.bdev = bdev; - src.sector = chunk_to_sector(s, pe->e.old_chunk); - src.count = min(s->chunk_size, dev_size - src.sector); + src.sector = chunk_to_sector(s->store, pe->e.old_chunk); + src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector); - dest.bdev = s->cow->bdev; - dest.sector = chunk_to_sector(s, pe->e.new_chunk); + dest.bdev = s->store->cow->bdev; + dest.sector = chunk_to_sector(s->store, pe->e.new_chunk); dest.count = src.count; /* Hand over to kcopyd */ - kcopyd_copy(s->kcopyd_client, + dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); } +static struct dm_snap_pending_exception * +__lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk) +{ + struct dm_snap_exception *e = lookup_exception(&s->pending, chunk); + + if (!e) + return NULL; + + return container_of(e, struct dm_snap_pending_exception, e); +} + /* * Looks to see if this snapshot already has a pending exception * for this chunk, otherwise it allocates a new one and inserts @@ -744,81 +991,83 @@ static void start_copy(struct pending_exception *pe) * NOTE: a write lock must be held on snap->lock before calling * this. */ -static struct pending_exception * -__find_pending_exception(struct dm_snapshot *s, struct bio *bio) +static struct dm_snap_pending_exception * +__find_pending_exception(struct dm_snapshot *s, + struct dm_snap_pending_exception *pe, chunk_t chunk) { - struct exception *e; - struct pending_exception *pe; - chunk_t chunk = sector_to_chunk(s, bio->bi_sector); + struct dm_snap_pending_exception *pe2; - /* - * Is there a pending exception for this already ? - */ - e = lookup_exception(&s->pending, chunk); - if (e) { - /* cast the exception to a pending exception */ - pe = container_of(e, struct pending_exception, e); - - } else { - /* - * Create a new pending exception, we don't want - * to hold the lock while we do this. - */ - up_write(&s->lock); - pe = alloc_pending_exception(); - down_write(&s->lock); + pe2 = __lookup_pending_exception(s, chunk); + if (pe2) { + free_pending_exception(pe); + return pe2; + } - e = lookup_exception(&s->pending, chunk); - if (e) { - free_pending_exception(pe); - pe = container_of(e, struct pending_exception, e); - } else { - pe->e.old_chunk = chunk; - bio_list_init(&pe->origin_bios); - bio_list_init(&pe->snapshot_bios); - pe->primary_pe = NULL; - atomic_set(&pe->sibling_count, 1); - pe->snap = s; - pe->started = 0; - - if (s->store.prepare_exception(&s->store, &pe->e)) { - free_pending_exception(pe); - s->valid = 0; - return NULL; - } + pe->e.old_chunk = chunk; + bio_list_init(&pe->origin_bios); + bio_list_init(&pe->snapshot_bios); + pe->primary_pe = NULL; + atomic_set(&pe->ref_count, 0); + pe->started = 0; - insert_exception(&s->pending, &pe->e); - } + if (s->store->type->prepare_exception(s->store, &pe->e)) { + free_pending_exception(pe); + return NULL; } + get_pending_exception(pe); + insert_exception(&s->pending, &pe->e); + return pe; } -static inline void remap_exception(struct dm_snapshot *s, struct exception *e, - struct bio *bio) +static void remap_exception(struct dm_snapshot *s, struct dm_snap_exception *e, + struct bio *bio, chunk_t chunk) { - bio->bi_bdev = s->cow->bdev; - bio->bi_sector = chunk_to_sector(s, e->new_chunk) + - (bio->bi_sector & s->chunk_mask); + bio->bi_bdev = s->store->cow->bdev; + bio->bi_sector = chunk_to_sector(s->store, + dm_chunk_number(e->new_chunk) + + (chunk - e->old_chunk)) + + (bio->bi_sector & + s->store->chunk_mask); } static int snapshot_map(struct dm_target *ti, struct bio *bio, union map_info *map_context) { - struct exception *e; - struct dm_snapshot *s = (struct dm_snapshot *) ti->private; - int r = 1; + struct dm_snap_exception *e; + struct dm_snapshot *s = ti->private; + int r = DM_MAPIO_REMAPPED; chunk_t chunk; - struct pending_exception *pe; + struct dm_snap_pending_exception *pe = NULL; - chunk = sector_to_chunk(s, bio->bi_sector); + if (unlikely(bio_empty_barrier(bio))) { + bio->bi_bdev = s->store->cow->bdev; + return DM_MAPIO_REMAPPED; + } + + chunk = sector_to_chunk(s->store, bio->bi_sector); /* Full snapshots are not usable */ + /* To get here the table must be live so s->active is always set. */ if (!s->valid) return -EIO; - if (unlikely(bio_barrier(bio))) - return -EOPNOTSUPP; + /* FIXME: should only take write lock if we need + * to copy an exception */ + down_write(&s->lock); + + if (!s->valid) { + r = -EIO; + goto out_unlock; + } + + /* If the block is already remapped - use that, else remap it */ + e = lookup_exception(&s->complete, chunk); + if (e) { + remap_exception(s, e, bio, chunk); + goto out_unlock; + } /* * Write to snapshot - higher level takes care of RW/RO @@ -826,68 +1075,71 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio, * writeable. */ if (bio_rw(bio) == WRITE) { - - /* FIXME: should only take write lock if we need - * to copy an exception */ - down_write(&s->lock); - - /* If the block is already remapped - use that, else remap it */ - e = lookup_exception(&s->complete, chunk); - if (e) { - remap_exception(s, e, bio); + pe = __lookup_pending_exception(s, chunk); + if (!pe) { up_write(&s->lock); + pe = alloc_pending_exception(s); + down_write(&s->lock); + + if (!s->valid) { + free_pending_exception(pe); + r = -EIO; + goto out_unlock; + } - } else { - pe = __find_pending_exception(s, bio); + e = lookup_exception(&s->complete, chunk); + if (e) { + free_pending_exception(pe); + remap_exception(s, e, bio, chunk); + goto out_unlock; + } + pe = __find_pending_exception(s, pe, chunk); if (!pe) { - if (s->store.drop_snapshot) - s->store.drop_snapshot(&s->store); - s->valid = 0; + __invalidate_snapshot(s, -ENOMEM); r = -EIO; - up_write(&s->lock); - } else { - remap_exception(s, &pe->e, bio); - bio_list_add(&pe->snapshot_bios, bio); - - if (!pe->started) { - /* this is protected by snap->lock */ - pe->started = 1; - up_write(&s->lock); - start_copy(pe); - } else - up_write(&s->lock); - r = 0; + goto out_unlock; } } - } else { - /* - * FIXME: this read path scares me because we - * always use the origin when we have a pending - * exception. However I can't think of a - * situation where this is wrong - ejt. - */ - - /* Do reads */ - down_read(&s->lock); + remap_exception(s, &pe->e, bio, chunk); + bio_list_add(&pe->snapshot_bios, bio); - /* See if it it has been remapped */ - e = lookup_exception(&s->complete, chunk); - if (e) - remap_exception(s, e, bio); - else - bio->bi_bdev = s->origin->bdev; + r = DM_MAPIO_SUBMITTED; - up_read(&s->lock); + if (!pe->started) { + /* this is protected by snap->lock */ + pe->started = 1; + up_write(&s->lock); + start_copy(pe); + goto out; + } + } else { + bio->bi_bdev = s->origin->bdev; + map_context->ptr = track_chunk(s, chunk); } + out_unlock: + up_write(&s->lock); + out: return r; } +static int snapshot_end_io(struct dm_target *ti, struct bio *bio, + int error, union map_info *map_context) +{ + struct dm_snapshot *s = ti->private; + struct dm_snap_tracked_chunk *c = map_context->ptr; + + if (c) + stop_tracking_chunk(s, c); + + return 0; +} + static void snapshot_resume(struct dm_target *ti) { - struct dm_snapshot *s = (struct dm_snapshot *) ti->private; + struct dm_snapshot *s = ti->private; down_write(&s->lock); s->active = 1; @@ -897,24 +1149,27 @@ static void snapshot_resume(struct dm_target *ti) static int snapshot_status(struct dm_target *ti, status_type_t type, char *result, unsigned int maxlen) { - struct dm_snapshot *snap = (struct dm_snapshot *) ti->private; + unsigned sz = 0; + struct dm_snapshot *snap = ti->private; + + down_write(&snap->lock); switch (type) { case STATUSTYPE_INFO: if (!snap->valid) - snprintf(result, maxlen, "Invalid"); + DMEMIT("Invalid"); else { - if (snap->store.fraction_full) { + if (snap->store->type->fraction_full) { sector_t numerator, denominator; - snap->store.fraction_full(&snap->store, - &numerator, - &denominator); - snprintf(result, maxlen, - SECTOR_FORMAT "/" SECTOR_FORMAT, - numerator, denominator); + snap->store->type->fraction_full(snap->store, + &numerator, + &denominator); + DMEMIT("%llu/%llu", + (unsigned long long)numerator, + (unsigned long long)denominator); } else - snprintf(result, maxlen, "Unknown"); + DMEMIT("Unknown"); } break; @@ -924,107 +1179,139 @@ static int snapshot_status(struct dm_target *ti, status_type_t type, * to make private copies if the output is to * make sense. */ - snprintf(result, maxlen, "%s %s %c " SECTOR_FORMAT, - snap->origin->name, snap->cow->name, - snap->type, snap->chunk_size); + DMEMIT("%s", snap->origin->name); + snap->store->type->status(snap->store, type, result + sz, + maxlen - sz); break; } + up_write(&snap->lock); + return 0; } +static int snapshot_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data) +{ + struct dm_snapshot *snap = ti->private; + + return fn(ti, snap->origin, 0, ti->len, data); +} + + /*----------------------------------------------------------------- * Origin methods *---------------------------------------------------------------*/ static int __origin_write(struct list_head *snapshots, struct bio *bio) { - int r = 1, first = 0; + int r = DM_MAPIO_REMAPPED, first = 0; struct dm_snapshot *snap; - struct exception *e; - struct pending_exception *pe, *next_pe, *primary_pe = NULL; + struct dm_snap_exception *e; + struct dm_snap_pending_exception *pe, *next_pe, *primary_pe = NULL; chunk_t chunk; LIST_HEAD(pe_queue); /* Do all the snapshots on this origin */ list_for_each_entry (snap, snapshots, list) { + down_write(&snap->lock); + /* Only deal with valid and active snapshots */ if (!snap->valid || !snap->active) - continue; + goto next_snapshot; /* Nothing to do if writing beyond end of snapshot */ - if (bio->bi_sector >= dm_table_get_size(snap->table)) - continue; - - down_write(&snap->lock); + if (bio->bi_sector >= dm_table_get_size(snap->store->ti->table)) + goto next_snapshot; /* * Remember, different snapshots can have * different chunk sizes. */ - chunk = sector_to_chunk(snap, bio->bi_sector); + chunk = sector_to_chunk(snap->store, bio->bi_sector); /* * Check exception table to see if block * is already remapped in this snapshot * and trigger an exception if not. * - * sibling_count is initialised to 1 so pending_complete() + * ref_count is initialised to 1 so pending_complete() * won't destroy the primary_pe while we're inside this loop. */ e = lookup_exception(&snap->complete, chunk); - if (!e) { - pe = __find_pending_exception(snap, bio); + if (e) + goto next_snapshot; + + pe = __lookup_pending_exception(snap, chunk); + if (!pe) { + up_write(&snap->lock); + pe = alloc_pending_exception(snap); + down_write(&snap->lock); + + if (!snap->valid) { + free_pending_exception(pe); + goto next_snapshot; + } + + e = lookup_exception(&snap->complete, chunk); + if (e) { + free_pending_exception(pe); + goto next_snapshot; + } + + pe = __find_pending_exception(snap, pe, chunk); if (!pe) { - snap->store.drop_snapshot(&snap->store); - snap->valid = 0; - - } else { - if (!primary_pe) { - /* - * Either every pe here has same - * primary_pe or none has one yet. - */ - if (pe->primary_pe) - primary_pe = pe->primary_pe; - else { - primary_pe = pe; - first = 1; - } - - bio_list_add(&primary_pe->origin_bios, - bio); - r = 0; - } - if (!pe->primary_pe) { - atomic_inc(&primary_pe->sibling_count); - pe->primary_pe = primary_pe; - } - if (!pe->started) { - pe->started = 1; - list_add_tail(&pe->list, &pe_queue); - } + __invalidate_snapshot(snap, -ENOMEM); + goto next_snapshot; } } + if (!primary_pe) { + /* + * Either every pe here has same + * primary_pe or none has one yet. + */ + if (pe->primary_pe) + primary_pe = pe->primary_pe; + else { + primary_pe = pe; + first = 1; + } + + bio_list_add(&primary_pe->origin_bios, bio); + + r = DM_MAPIO_SUBMITTED; + } + + if (!pe->primary_pe) { + pe->primary_pe = primary_pe; + get_pending_exception(primary_pe); + } + + if (!pe->started) { + pe->started = 1; + list_add_tail(&pe->list, &pe_queue); + } + + next_snapshot: up_write(&snap->lock); } if (!primary_pe) - goto out; + return r; /* * If this is the first time we're processing this chunk and - * sibling_count is now 1 it means all the pending exceptions + * ref_count is now 1 it means all the pending exceptions * got completed while we were in the loop above, so it falls to * us here to remove the primary_pe and submit any origin_bios. */ - if (first && atomic_dec_and_test(&primary_pe->sibling_count)) { + if (first && atomic_dec_and_test(&primary_pe->ref_count)) { flush_bios(bio_list_get(&primary_pe->origin_bios)); free_pending_exception(primary_pe); /* If we got here, pe_queue is necessarily empty. */ - goto out; + return r; } /* @@ -1033,7 +1320,6 @@ static int __origin_write(struct list_head *snapshots, struct bio *bio) list_for_each_entry_safe(pe, next_pe, &pe_queue, list) start_copy(pe); - out: return r; } @@ -1043,7 +1329,7 @@ static int __origin_write(struct list_head *snapshots, struct bio *bio) static int do_origin(struct dm_dev *origin, struct bio *bio) { struct origin *o; - int r = 1; + int r = DM_MAPIO_REMAPPED; down_read(&_origins_lock); o = __lookup_origin(origin->bdev); @@ -1069,7 +1355,7 @@ static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv) struct dm_dev *dev; if (argc != 1) { - ti->error = "dm-origin: incorrect number of arguments"; + ti->error = "origin: incorrect number of arguments"; return -EINVAL; } @@ -1081,26 +1367,28 @@ static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv) } ti->private = dev; + ti->num_flush_requests = 1; + return 0; } static void origin_dtr(struct dm_target *ti) { - struct dm_dev *dev = (struct dm_dev *) ti->private; + struct dm_dev *dev = ti->private; dm_put_device(ti, dev); } static int origin_map(struct dm_target *ti, struct bio *bio, union map_info *map_context) { - struct dm_dev *dev = (struct dm_dev *) ti->private; + struct dm_dev *dev = ti->private; bio->bi_bdev = dev->bdev; - if (unlikely(bio_barrier(bio))) - return -EOPNOTSUPP; + if (unlikely(bio_empty_barrier(bio))) + return DM_MAPIO_REMAPPED; /* Only tell snapshots if this is a write */ - return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : 1; + return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED; } #define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r)) @@ -1111,16 +1399,17 @@ static int origin_map(struct dm_target *ti, struct bio *bio, */ static void origin_resume(struct dm_target *ti) { - struct dm_dev *dev = (struct dm_dev *) ti->private; + struct dm_dev *dev = ti->private; struct dm_snapshot *snap; struct origin *o; - chunk_t chunk_size = 0; + unsigned chunk_size = 0; down_read(&_origins_lock); o = __lookup_origin(dev->bdev); if (o) list_for_each_entry (snap, &o->snapshots, list) - chunk_size = min_not_zero(chunk_size, snap->chunk_size); + chunk_size = min_not_zero(chunk_size, + snap->store->chunk_size); up_read(&_origins_lock); ti->split_io = chunk_size; @@ -1129,7 +1418,7 @@ static void origin_resume(struct dm_target *ti) static int origin_status(struct dm_target *ti, status_type_t type, char *result, unsigned int maxlen) { - struct dm_dev *dev = (struct dm_dev *) ti->private; + struct dm_dev *dev = ti->private; switch (type) { case STATUSTYPE_INFO: @@ -1144,41 +1433,58 @@ static int origin_status(struct dm_target *ti, status_type_t type, char *result, return 0; } +static int origin_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data) +{ + struct dm_dev *dev = ti->private; + + return fn(ti, dev, 0, ti->len, data); +} + static struct target_type origin_target = { .name = "snapshot-origin", - .version = {1, 1, 0}, + .version = {1, 7, 0}, .module = THIS_MODULE, .ctr = origin_ctr, .dtr = origin_dtr, .map = origin_map, .resume = origin_resume, .status = origin_status, + .iterate_devices = origin_iterate_devices, }; static struct target_type snapshot_target = { .name = "snapshot", - .version = {1, 1, 0}, + .version = {1, 7, 0}, .module = THIS_MODULE, .ctr = snapshot_ctr, .dtr = snapshot_dtr, .map = snapshot_map, + .end_io = snapshot_end_io, .resume = snapshot_resume, .status = snapshot_status, + .iterate_devices = snapshot_iterate_devices, }; static int __init dm_snapshot_init(void) { int r; + r = dm_exception_store_init(); + if (r) { + DMERR("Failed to initialize exception stores"); + return r; + } + r = dm_register_target(&snapshot_target); if (r) { DMERR("snapshot target register failed %d", r); - return r; + goto bad_register_snapshot_target; } r = dm_register_target(&origin_target); if (r < 0) { - DMERR("Device mapper: Origin: register failed %d\n", r); + DMERR("Origin target register failed %d", r); goto bad1; } @@ -1188,65 +1494,67 @@ static int __init dm_snapshot_init(void) goto bad2; } - exception_cache = kmem_cache_create("dm-snapshot-ex", - sizeof(struct exception), - __alignof__(struct exception), - 0, NULL, NULL); + exception_cache = KMEM_CACHE(dm_snap_exception, 0); if (!exception_cache) { DMERR("Couldn't create exception cache."); r = -ENOMEM; goto bad3; } - pending_cache = - kmem_cache_create("dm-snapshot-in", - sizeof(struct pending_exception), - __alignof__(struct pending_exception), - 0, NULL, NULL); + pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0); if (!pending_cache) { DMERR("Couldn't create pending cache."); r = -ENOMEM; goto bad4; } - pending_pool = mempool_create_slab_pool(128, pending_cache); - if (!pending_pool) { - DMERR("Couldn't create pending pool."); + tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0); + if (!tracked_chunk_cache) { + DMERR("Couldn't create cache to track chunks in use."); r = -ENOMEM; goto bad5; } + ksnapd = create_singlethread_workqueue("ksnapd"); + if (!ksnapd) { + DMERR("Failed to create ksnapd workqueue."); + r = -ENOMEM; + goto bad_pending_pool; + } + return 0; - bad5: +bad_pending_pool: + kmem_cache_destroy(tracked_chunk_cache); +bad5: kmem_cache_destroy(pending_cache); - bad4: +bad4: kmem_cache_destroy(exception_cache); - bad3: +bad3: exit_origin_hash(); - bad2: +bad2: dm_unregister_target(&origin_target); - bad1: +bad1: dm_unregister_target(&snapshot_target); + +bad_register_snapshot_target: + dm_exception_store_exit(); return r; } static void __exit dm_snapshot_exit(void) { - int r; - - r = dm_unregister_target(&snapshot_target); - if (r) - DMERR("snapshot unregister failed %d", r); + destroy_workqueue(ksnapd); - r = dm_unregister_target(&origin_target); - if (r) - DMERR("origin unregister failed %d", r); + dm_unregister_target(&snapshot_target); + dm_unregister_target(&origin_target); exit_origin_hash(); - mempool_destroy(pending_pool); kmem_cache_destroy(pending_cache); kmem_cache_destroy(exception_cache); + kmem_cache_destroy(tracked_chunk_cache); + + dm_exception_store_exit(); } /* Module hooks */