* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
- *
- * $Id: fmr_pool.c 2730 2005-06-28 16:43:03Z sean.hefty $
*/
#include <linux/errno.h>
#include <linux/jhash.h>
#include <linux/kthread.h>
-#include <ib_fmr_pool.h>
+#include <rdma/ib_fmr_pool.h>
#include "core_priv.h"
+#define PFX "fmr_pool: "
+
enum {
IB_FMR_MAX_REMAPS = 32,
/*
* If an FMR is not in use, then the list member will point to either
* its pool's free_list (if the FMR can be mapped again; that is,
- * remap_count < IB_FMR_MAX_REMAPS) or its pool's dirty_list (if the
+ * remap_count < pool->max_remaps) or its pool's dirty_list (if the
* FMR needs to be unmapped before being remapped). In either of
* these cases it is a bug if the ref_count is not 0. In other words,
* if ref_count is > 0, then the list member must not be linked into
int pool_size;
int max_pages;
+ int max_remaps;
int dirty_watermark;
int dirty_len;
struct list_head free_list;
#ifdef DEBUG
if (fmr->ref_count !=0) {
- printk(KERN_WARNING "Unmapping FMR 0x%08x with ref count %d",
+ printk(KERN_WARNING PFX "Unmapping FMR 0x%08x with ref count %d\n",
fmr, fmr->ref_count);
}
#endif
}
- list_splice(&pool->dirty_list, &unmap_list);
- INIT_LIST_HEAD(&pool->dirty_list);
+ list_splice_init(&pool->dirty_list, &unmap_list);
pool->dirty_len = 0;
spin_unlock_irq(&pool->pool_lock);
ret = ib_unmap_fmr(&fmr_list);
if (ret)
- printk(KERN_WARNING "ib_unmap_fmr returned %d", ret);
+ printk(KERN_WARNING PFX "ib_unmap_fmr returned %d\n", ret);
spin_lock_irq(&pool->pool_lock);
list_splice(&unmap_list, &pool->free_list);
struct ib_fmr_pool *pool = pool_ptr;
do {
- if (pool->dirty_len >= pool->dirty_watermark ||
- atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
+ if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
ib_fmr_batch_release(pool);
atomic_inc(&pool->flush_ser);
}
set_current_state(TASK_INTERRUPTIBLE);
- if (pool->dirty_len < pool->dirty_watermark &&
- atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
+ if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
!kthread_should_stop())
schedule();
__set_current_state(TASK_RUNNING);
{
struct ib_device *device;
struct ib_fmr_pool *pool;
+ struct ib_device_attr *attr;
int i;
int ret;
+ int max_remaps;
if (!params)
return ERR_PTR(-EINVAL);
device = pd->device;
if (!device->alloc_fmr || !device->dealloc_fmr ||
!device->map_phys_fmr || !device->unmap_fmr) {
- printk(KERN_WARNING "Device %s does not support fast memory regions",
+ printk(KERN_INFO PFX "Device %s does not support FMRs\n",
device->name);
return ERR_PTR(-ENOSYS);
}
+ attr = kmalloc(sizeof *attr, GFP_KERNEL);
+ if (!attr) {
+ printk(KERN_WARNING PFX "couldn't allocate device attr struct\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ret = ib_query_device(device, attr);
+ if (ret) {
+ printk(KERN_WARNING PFX "couldn't query device: %d\n", ret);
+ kfree(attr);
+ return ERR_PTR(ret);
+ }
+
+ if (!attr->max_map_per_fmr)
+ max_remaps = IB_FMR_MAX_REMAPS;
+ else
+ max_remaps = attr->max_map_per_fmr;
+
+ kfree(attr);
+
pool = kmalloc(sizeof *pool, GFP_KERNEL);
if (!pool) {
- printk(KERN_WARNING "couldn't allocate pool struct");
+ printk(KERN_WARNING PFX "couldn't allocate pool struct\n");
return ERR_PTR(-ENOMEM);
}
kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,
GFP_KERNEL);
if (!pool->cache_bucket) {
- printk(KERN_WARNING "Failed to allocate cache in pool");
+ printk(KERN_WARNING PFX "Failed to allocate cache in pool\n");
ret = -ENOMEM;
goto out_free_pool;
}
pool->pool_size = 0;
pool->max_pages = params->max_pages_per_fmr;
+ pool->max_remaps = max_remaps;
pool->dirty_watermark = params->dirty_watermark;
pool->dirty_len = 0;
spin_lock_init(&pool->pool_lock);
atomic_set(&pool->flush_ser, 0);
init_waitqueue_head(&pool->force_wait);
- pool->thread = kthread_create(ib_fmr_cleanup_thread,
- pool,
- "ib_fmr(%s)",
- device->name);
+ pool->thread = kthread_run(ib_fmr_cleanup_thread,
+ pool,
+ "ib_fmr(%s)",
+ device->name);
if (IS_ERR(pool->thread)) {
- printk(KERN_WARNING "couldn't start cleanup thread");
+ printk(KERN_WARNING PFX "couldn't start cleanup thread\n");
ret = PTR_ERR(pool->thread);
goto out_free_pool;
}
{
struct ib_pool_fmr *fmr;
- struct ib_fmr_attr attr = {
- .max_pages = params->max_pages_per_fmr,
- .max_maps = IB_FMR_MAX_REMAPS,
- .page_size = PAGE_SHIFT
+ struct ib_fmr_attr fmr_attr = {
+ .max_pages = params->max_pages_per_fmr,
+ .max_maps = pool->max_remaps,
+ .page_shift = params->page_shift
};
+ int bytes_per_fmr = sizeof *fmr;
+
+ if (pool->cache_bucket)
+ bytes_per_fmr += params->max_pages_per_fmr * sizeof (u64);
for (i = 0; i < params->pool_size; ++i) {
- fmr = kmalloc(sizeof *fmr + params->max_pages_per_fmr * sizeof (u64),
- GFP_KERNEL);
+ fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
if (!fmr) {
- printk(KERN_WARNING "failed to allocate fmr struct "
- "for FMR %d", i);
+ printk(KERN_WARNING PFX "failed to allocate fmr "
+ "struct for FMR %d\n", i);
goto out_fail;
}
fmr->ref_count = 0;
INIT_HLIST_NODE(&fmr->cache_node);
- fmr->fmr = ib_alloc_fmr(pd, params->access, &attr);
+ fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
if (IS_ERR(fmr->fmr)) {
- printk(KERN_WARNING "fmr_create failed for FMR %d", i);
+ printk(KERN_WARNING PFX "fmr_create failed "
+ "for FMR %d\n", i);
kfree(fmr);
goto out_fail;
}
{
struct ib_pool_fmr *fmr;
struct ib_pool_fmr *tmp;
+ LIST_HEAD(fmr_list);
int i;
kthread_stop(pool->thread);
i = 0;
list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
+ if (fmr->remap_count) {
+ INIT_LIST_HEAD(&fmr_list);
+ list_add_tail(&fmr->fmr->list, &fmr_list);
+ ib_unmap_fmr(&fmr_list);
+ }
ib_dealloc_fmr(fmr->fmr);
list_del(&fmr->list);
kfree(fmr);
}
if (i < pool->pool_size)
- printk(KERN_WARNING "pool still has %d regions registered",
+ printk(KERN_WARNING PFX "pool still has %d regions registered\n",
pool->pool_size - i);
kfree(pool->cache_bucket);
int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
{
int serial;
+ struct ib_pool_fmr *fmr, *next;
- atomic_inc(&pool->req_ser);
/*
- * It's OK if someone else bumps req_ser again here -- we'll
- * just wait a little longer.
+ * The free_list holds FMRs that may have been used
+ * but have not been remapped enough times to be dirty.
+ * Put them on the dirty list now so that the cleanup
+ * thread will reap them too.
*/
- serial = atomic_read(&pool->req_ser);
+ spin_lock_irq(&pool->pool_lock);
+ list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
+ if (fmr->remap_count > 0)
+ list_move(&fmr->list, &pool->dirty_list);
+ }
+ spin_unlock_irq(&pool->pool_lock);
+ serial = atomic_inc_return(&pool->req_ser);
wake_up_process(pool->thread);
if (wait_event_interruptible(pool->force_wait,
- atomic_read(&pool->flush_ser) -
- atomic_read(&pool->req_ser) >= 0))
+ atomic_read(&pool->flush_ser) - serial >= 0))
return -EINTR;
return 0;
struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
u64 *page_list,
int list_len,
- u64 *io_virtual_address)
+ u64 io_virtual_address)
{
struct ib_fmr_pool *pool = pool_handle;
struct ib_pool_fmr *fmr;
fmr = ib_fmr_cache_lookup(pool,
page_list,
list_len,
- *io_virtual_address);
+ io_virtual_address);
if (fmr) {
/* found in cache */
++fmr->ref_count;
spin_unlock_irqrestore(&pool->pool_lock, flags);
result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
- *io_virtual_address);
+ io_virtual_address);
if (result) {
spin_lock_irqsave(&pool->pool_lock, flags);
list_add(&fmr->list, &pool->free_list);
spin_unlock_irqrestore(&pool->pool_lock, flags);
- printk(KERN_WARNING "fmr_map returns %d\n",
- result);
+ printk(KERN_WARNING PFX "fmr_map returns %d\n", result);
return ERR_PTR(result);
}
fmr->ref_count = 1;
if (pool->cache_bucket) {
- fmr->io_virtual_address = *io_virtual_address;
+ fmr->io_virtual_address = io_virtual_address;
fmr->page_list_len = list_len;
memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));
--fmr->ref_count;
if (!fmr->ref_count) {
- if (fmr->remap_count < IB_FMR_MAX_REMAPS) {
+ if (fmr->remap_count < pool->max_remaps) {
list_add_tail(&fmr->list, &pool->free_list);
} else {
list_add_tail(&fmr->list, &pool->dirty_list);
- ++pool->dirty_len;
- wake_up_process(pool->thread);
+ if (++pool->dirty_len >= pool->dirty_watermark) {
+ atomic_inc(&pool->req_ser);
+ wake_up_process(pool->thread);
+ }
}
}
#ifdef DEBUG
if (fmr->ref_count < 0)
- printk(KERN_WARNING "FMR %p has ref count %d < 0",
+ printk(KERN_WARNING PFX "FMR %p has ref count %d < 0\n",
fmr, fmr->ref_count);
#endif
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff