drivers/infiniband/hw/mlx4/mr.c - kernel/bruno - Git at Google

 /*
  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 #include <linux/slab.h>

 #include "mlx4_ib.h"

 static u32 convert_access(int acc)
 {
 	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC       : 0) |
 	       (acc & IB_ACCESS_REMOTE_WRITE  ? MLX4_PERM_REMOTE_WRITE : 0) |
 	       (acc & IB_ACCESS_REMOTE_READ   ? MLX4_PERM_REMOTE_READ  : 0) |
 	       (acc & IB_ACCESS_LOCAL_WRITE   ? MLX4_PERM_LOCAL_WRITE  : 0) |
 	       (acc & IB_ACCESS_MW_BIND	      ? MLX4_PERM_BIND_MW      : 0) |
 	       MLX4_PERM_LOCAL_READ;
 }

 static enum mlx4_mw_type to_mlx4_type(enum ib_mw_type type)
 {
 	switch (type) {
 	case IB_MW_TYPE_1:	return MLX4_MW_TYPE_1;
 	case IB_MW_TYPE_2:	return MLX4_MW_TYPE_2;
 	default:		return -1;
 	}
 }

 struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc)
 {
 	struct mlx4_ib_mr *mr;
 	int err;

 	mr = kmalloc(sizeof *mr, GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);

 	err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0,
 			    ~0ull, convert_access(acc), 0, 0, &mr->mmr);
 	if (err)
 		goto err_free;

 	err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr);
 	if (err)
 		goto err_mr;

 	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
 	mr->umem = NULL;

 	return &mr->ibmr;

 err_mr:
 	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr);

 err_free:
 	kfree(mr);

 	return ERR_PTR(err);
 }

 int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
 			   struct ib_umem *umem)
 {
 	u64 *pages;
 	int i, k, entry;
 	int n;
 	int len;
 	int err = 0;
 	struct scatterlist *sg;

 	pages = (u64 *) __get_free_page(GFP_KERNEL);
 	if (!pages)
 		return -ENOMEM;

 	i = n = 0;

 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
 		len = sg_dma_len(sg) >> mtt->page_shift;
 		for (k = 0; k < len; ++k) {
 			pages[i++] = sg_dma_address(sg) +
 				umem->page_size * k;
 			/*
 			 * Be friendly to mlx4_write_mtt() and
 			 * pass it chunks of appropriate size.
 			 */
 			if (i == PAGE_SIZE / sizeof (u64)) {
 				err = mlx4_write_mtt(dev->dev, mtt, n,
 						     i, pages);
 				if (err)
 					goto out;
 				n += i;
 				i = 0;
 			}
 		}
 	}

 	if (i)
 		err = mlx4_write_mtt(dev->dev, mtt, n, i, pages);

 out:
 	free_page((unsigned long) pages);
 	return err;
 }

 struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				  u64 virt_addr, int access_flags,
 				  struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_mr *mr;
 	int shift;
 	int err;
 	int n;

 	mr = kmalloc(sizeof *mr, GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);

 	/* Force registering the memory as writable. */
 	/* Used for memory re-registeration. HCA protects the access */
 	mr->umem = ib_umem_get(pd->uobject->context, start, length,
 			       access_flags | IB_ACCESS_LOCAL_WRITE, 0);
 	if (IS_ERR(mr->umem)) {
 		err = PTR_ERR(mr->umem);
 		goto err_free;
 	}

 	n = ib_umem_page_count(mr->umem);
 	shift = ilog2(mr->umem->page_size);

 	err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length,
 			    convert_access(access_flags), n, shift, &mr->mmr);
 	if (err)
 		goto err_umem;

 	err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem);
 	if (err)
 		goto err_mr;

 	err = mlx4_mr_enable(dev->dev, &mr->mmr);
 	if (err)
 		goto err_mr;

 	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;

 	return &mr->ibmr;

 err_mr:
 	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr);

 err_umem:
 	ib_umem_release(mr->umem);

 err_free:
 	kfree(mr);

 	return ERR_PTR(err);
 }

 int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
 			  u64 start, u64 length, u64 virt_addr,
 			  int mr_access_flags, struct ib_pd *pd,
 			  struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(mr->device);
 	struct mlx4_ib_mr *mmr = to_mmr(mr);
 	struct mlx4_mpt_entry *mpt_entry;
 	struct mlx4_mpt_entry **pmpt_entry = &mpt_entry;
 	int err;

 	/* Since we synchronize this call and mlx4_ib_dereg_mr via uverbs,
 	 * we assume that the calls can't run concurrently. Otherwise, a
 	 * race exists.
 	 */
 	err =  mlx4_mr_hw_get_mpt(dev->dev, &mmr->mmr, &pmpt_entry);

 	if (err)
 		return err;

 	if (flags & IB_MR_REREG_PD) {
 		err = mlx4_mr_hw_change_pd(dev->dev, *pmpt_entry,
 					   to_mpd(pd)->pdn);

 		if (err)
 			goto release_mpt_entry;
 	}

 	if (flags & IB_MR_REREG_ACCESS) {
 		err = mlx4_mr_hw_change_access(dev->dev, *pmpt_entry,
 					       convert_access(mr_access_flags));

 		if (err)
 			goto release_mpt_entry;
 	}

 	if (flags & IB_MR_REREG_TRANS) {
 		int shift;
 		int n;

 		mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr);
 		ib_umem_release(mmr->umem);
 		mmr->umem = ib_umem_get(mr->uobject->context, start, length,
 					mr_access_flags |
 					IB_ACCESS_LOCAL_WRITE,
 					0);
 		if (IS_ERR(mmr->umem)) {
 			err = PTR_ERR(mmr->umem);
 			/* Prevent mlx4_ib_dereg_mr from free'ing invalid pointer */
 			mmr->umem = NULL;
 			goto release_mpt_entry;
 		}
 		n = ib_umem_page_count(mmr->umem);
 		shift = ilog2(mmr->umem->page_size);

 		err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr,
 					      virt_addr, length, n, shift,
 					      *pmpt_entry);
 		if (err) {
 			ib_umem_release(mmr->umem);
 			goto release_mpt_entry;
 		}
 		mmr->mmr.iova       = virt_addr;
 		mmr->mmr.size       = length;

 		err = mlx4_ib_umem_write_mtt(dev, &mmr->mmr.mtt, mmr->umem);
 		if (err) {
 			mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr);
 			ib_umem_release(mmr->umem);
 			goto release_mpt_entry;
 		}
 	}

 	/* If we couldn't transfer the MR to the HCA, just remember to
 	 * return a failure. But dereg_mr will free the resources.
 	 */
 	err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry);
 	if (!err && flags & IB_MR_REREG_ACCESS)
 		mmr->mmr.access = mr_access_flags;

 release_mpt_entry:
 	mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry);

 	return err;
 }

 int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
 {
 	struct mlx4_ib_mr *mr = to_mmr(ibmr);
 	int ret;

 	ret = mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr);
 	if (ret)
 		return ret;
 	if (mr->umem)
 		ib_umem_release(mr->umem);
 	kfree(mr);

 	return 0;
 }

 struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
 {
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_mw *mw;
 	int err;

 	mw = kmalloc(sizeof(*mw), GFP_KERNEL);
 	if (!mw)
 		return ERR_PTR(-ENOMEM);

 	err = mlx4_mw_alloc(dev->dev, to_mpd(pd)->pdn,
 			    to_mlx4_type(type), &mw->mmw);
 	if (err)
 		goto err_free;

 	err = mlx4_mw_enable(dev->dev, &mw->mmw);
 	if (err)
 		goto err_mw;

 	mw->ibmw.rkey = mw->mmw.key;

 	return &mw->ibmw;

 err_mw:
 	mlx4_mw_free(dev->dev, &mw->mmw);

 err_free:
 	kfree(mw);

 	return ERR_PTR(err);
 }

 int mlx4_ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
 		    struct ib_mw_bind *mw_bind)
 {
 	struct ib_send_wr  wr;
 	struct ib_send_wr *bad_wr;
 	int ret;

 	memset(&wr, 0, sizeof(wr));
 	wr.opcode               = IB_WR_BIND_MW;
 	wr.wr_id                = mw_bind->wr_id;
 	wr.send_flags           = mw_bind->send_flags;
 	wr.wr.bind_mw.mw        = mw;
 	wr.wr.bind_mw.bind_info = mw_bind->bind_info;
 	wr.wr.bind_mw.rkey      = ib_inc_rkey(mw->rkey);

 	ret = mlx4_ib_post_send(qp, &wr, &bad_wr);
 	if (!ret)
 		mw->rkey = wr.wr.bind_mw.rkey;

 	return ret;
 }

 int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)
 {
 	struct mlx4_ib_mw *mw = to_mmw(ibmw);

 	mlx4_mw_free(to_mdev(ibmw->device)->dev, &mw->mmw);
 	kfree(mw);

 	return 0;
 }

 struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd,
 					int max_page_list_len)
 {
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_mr *mr;
 	int err;

 	mr = kmalloc(sizeof *mr, GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);

 	err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0,
 			    max_page_list_len, 0, &mr->mmr);
 	if (err)
 		goto err_free;

 	err = mlx4_mr_enable(dev->dev, &mr->mmr);
 	if (err)
 		goto err_mr;

 	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
 	mr->umem = NULL;

 	return &mr->ibmr;

 err_mr:
 	(void) mlx4_mr_free(dev->dev, &mr->mmr);

 err_free:
 	kfree(mr);
 	return ERR_PTR(err);
 }

 struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
 							       int page_list_len)
 {
 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
 	struct mlx4_ib_fast_reg_page_list *mfrpl;
 	int size = page_list_len * sizeof (u64);

 	if (page_list_len > MLX4_MAX_FAST_REG_PAGES)
 		return ERR_PTR(-EINVAL);

 	mfrpl = kmalloc(sizeof *mfrpl, GFP_KERNEL);
 	if (!mfrpl)
 		return ERR_PTR(-ENOMEM);

 	mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL);
 	if (!mfrpl->ibfrpl.page_list)
 		goto err_free;

 	mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev,
 						     size, &mfrpl->map,
 						     GFP_KERNEL);
 	if (!mfrpl->mapped_page_list)
 		goto err_free;

 	WARN_ON(mfrpl->map & 0x3f);

 	return &mfrpl->ibfrpl;

 err_free:
 	kfree(mfrpl->ibfrpl.page_list);
 	kfree(mfrpl);
 	return ERR_PTR(-ENOMEM);
 }

 void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
 {
 	struct mlx4_ib_dev *dev = to_mdev(page_list->device);
 	struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list);
 	int size = page_list->max_page_list_len * sizeof (u64);

 	dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list,
 			  mfrpl->map);
 	kfree(mfrpl->ibfrpl.page_list);
 	kfree(mfrpl);
 }

 struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc,
 				 struct ib_fmr_attr *fmr_attr)
 {
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_fmr *fmr;
 	int err = -ENOMEM;

 	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
 	if (!fmr)
 		return ERR_PTR(-ENOMEM);

 	err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc),
 			     fmr_attr->max_pages, fmr_attr->max_maps,
 			     fmr_attr->page_shift, &fmr->mfmr);
 	if (err)
 		goto err_free;

 	err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr);
 	if (err)
 		goto err_mr;

 	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key;

 	return &fmr->ibfmr;

 err_mr:
 	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr);

 err_free:
 	kfree(fmr);

 	return ERR_PTR(err);
 }

 int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
 		      int npages, u64 iova)
 {
 	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
 	struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);

 	return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,
 				 &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
 }

 int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
 {
 	struct ib_fmr *ibfmr;
 	int err;
 	struct mlx4_dev *mdev = NULL;

 	list_for_each_entry(ibfmr, fmr_list, list) {
 		if (mdev && to_mdev(ibfmr->device)->dev != mdev)
 			return -EINVAL;
 		mdev = to_mdev(ibfmr->device)->dev;
 	}

 	if (!mdev)
 		return 0;

 	list_for_each_entry(ibfmr, fmr_list, list) {
 		struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);

 		mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
 	}

 	/*
 	 * Make sure all MPT status updates are visible before issuing
 	 * SYNC_TPT firmware command.
 	 */
 	wmb();

 	err = mlx4_SYNC_TPT(mdev);
 	if (err)
 		pr_warn("SYNC_TPT error %d when "
 		       "unmapping FMRs\n", err);

 	return 0;
 }

 int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)
 {
 	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
 	struct mlx4_ib_dev *dev = to_mdev(ibfmr->device);
 	int err;

 	err = mlx4_fmr_free(dev->dev, &ifmr->mfmr);

 	if (!err)
 		kfree(ifmr);

 	return err;
 }
	/*
	* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
	* Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
	*
	* This software is available to you under a choice of one of two
	* licenses. You may choose to be licensed under the terms of the GNU
	* General Public License (GPL) Version 2, available from the file
	* COPYING in the main directory of this source tree, or the
	* OpenIB.org BSD license below:
	*
	* Redistribution and use in source and binary forms, with or
	* without modification, are permitted provided that the following
	* conditions are met:
	*
	* - Redistributions of source code must retain the above
	* copyright notice, this list of conditions and the following
	* disclaimer.
	*
	* - Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following
	* disclaimer in the documentation and/or other materials
	* provided with the distribution.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*/

	#include <linux/slab.h>

	#include "mlx4_ib.h"

	static u32 convert_access(int acc)
	{
	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC : 0) \|
	(acc & IB_ACCESS_REMOTE_WRITE ? MLX4_PERM_REMOTE_WRITE : 0) \|
	(acc & IB_ACCESS_REMOTE_READ ? MLX4_PERM_REMOTE_READ : 0) \|
	(acc & IB_ACCESS_LOCAL_WRITE ? MLX4_PERM_LOCAL_WRITE : 0) \|
	(acc & IB_ACCESS_MW_BIND ? MLX4_PERM_BIND_MW : 0) \|
	MLX4_PERM_LOCAL_READ;
	}

	static enum mlx4_mw_type to_mlx4_type(enum ib_mw_type type)
	{
	switch (type) {
	case IB_MW_TYPE_1: return MLX4_MW_TYPE_1;
	case IB_MW_TYPE_2: return MLX4_MW_TYPE_2;
	default: return -1;
	}
	}

	struct ib_mr mlx4_ib_get_dma_mr(struct ib_pd pd, int acc)
	{
	struct mlx4_ib_mr *mr;
	int err;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
	return ERR_PTR(-ENOMEM);

	err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0,
	~0ull, convert_access(acc), 0, 0, &mr->mmr);
	if (err)
	goto err_free;

	err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr);
	if (err)
	goto err_mr;

	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
	mr->umem = NULL;

	return &mr->ibmr;

	err_mr:
	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr);

	err_free:
	kfree(mr);

	return ERR_PTR(err);
	}

	int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev dev, struct mlx4_mtt mtt,
	struct ib_umem *umem)
	{
	u64 *pages;
	int i, k, entry;
	int n;
	int len;
	int err = 0;
	struct scatterlist *sg;

	pages = (u64 *) __get_free_page(GFP_KERNEL);
	if (!pages)
	return -ENOMEM;

	i = n = 0;

	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
	len = sg_dma_len(sg) >> mtt->page_shift;
	for (k = 0; k < len; ++k) {
	pages[i++] = sg_dma_address(sg) +
	umem->page_size * k;
	/*
	* Be friendly to mlx4_write_mtt() and
	* pass it chunks of appropriate size.
	*/
	if (i == PAGE_SIZE / sizeof (u64)) {
	err = mlx4_write_mtt(dev->dev, mtt, n,
	i, pages);
	if (err)
	goto out;
	n += i;
	i = 0;
	}
	}
	}

	if (i)
	err = mlx4_write_mtt(dev->dev, mtt, n, i, pages);

	out:
	free_page((unsigned long) pages);
	return err;
	}

	struct ib_mr mlx4_ib_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
	u64 virt_addr, int access_flags,
	struct ib_udata *udata)
	{
	struct mlx4_ib_dev *dev = to_mdev(pd->device);
	struct mlx4_ib_mr *mr;
	int shift;
	int err;
	int n;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
	return ERR_PTR(-ENOMEM);

	/* Force registering the memory as writable. */
	/* Used for memory re-registeration. HCA protects the access */
	mr->umem = ib_umem_get(pd->uobject->context, start, length,
	access_flags \| IB_ACCESS_LOCAL_WRITE, 0);
	if (IS_ERR(mr->umem)) {
	err = PTR_ERR(mr->umem);
	goto err_free;
	}

	n = ib_umem_page_count(mr->umem);
	shift = ilog2(mr->umem->page_size);

	err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length,
	convert_access(access_flags), n, shift, &mr->mmr);
	if (err)
	goto err_umem;

	err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem);
	if (err)
	goto err_mr;

	err = mlx4_mr_enable(dev->dev, &mr->mmr);
	if (err)
	goto err_mr;

	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;

	return &mr->ibmr;

	err_mr:
	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr);

	err_umem:
	ib_umem_release(mr->umem);

	err_free:
	kfree(mr);

	return ERR_PTR(err);
	}

	int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
	u64 start, u64 length, u64 virt_addr,
	int mr_access_flags, struct ib_pd *pd,
	struct ib_udata *udata)
	{
	struct mlx4_ib_dev *dev = to_mdev(mr->device);
	struct mlx4_ib_mr *mmr = to_mmr(mr);
	struct mlx4_mpt_entry *mpt_entry;
	struct mlx4_mpt_entry **pmpt_entry = &mpt_entry;
	int err;

	/* Since we synchronize this call and mlx4_ib_dereg_mr via uverbs,
	* we assume that the calls can't run concurrently. Otherwise, a
	* race exists.
	*/
	err = mlx4_mr_hw_get_mpt(dev->dev, &mmr->mmr, &pmpt_entry);

	if (err)
	return err;

	if (flags & IB_MR_REREG_PD) {
	err = mlx4_mr_hw_change_pd(dev->dev, *pmpt_entry,
	to_mpd(pd)->pdn);

	if (err)
	goto release_mpt_entry;
	}

	if (flags & IB_MR_REREG_ACCESS) {
	err = mlx4_mr_hw_change_access(dev->dev, *pmpt_entry,
	convert_access(mr_access_flags));

	if (err)
	goto release_mpt_entry;
	}

	if (flags & IB_MR_REREG_TRANS) {
	int shift;
	int n;

	mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr);
	ib_umem_release(mmr->umem);
	mmr->umem = ib_umem_get(mr->uobject->context, start, length,
	mr_access_flags \|
	IB_ACCESS_LOCAL_WRITE,
	0);
	if (IS_ERR(mmr->umem)) {
	err = PTR_ERR(mmr->umem);
	/* Prevent mlx4_ib_dereg_mr from free'ing invalid pointer */
	mmr->umem = NULL;
	goto release_mpt_entry;
	}
	n = ib_umem_page_count(mmr->umem);
	shift = ilog2(mmr->umem->page_size);

	err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr,
	virt_addr, length, n, shift,
	*pmpt_entry);
	if (err) {
	ib_umem_release(mmr->umem);
	goto release_mpt_entry;
	}
	mmr->mmr.iova = virt_addr;
	mmr->mmr.size = length;

	err = mlx4_ib_umem_write_mtt(dev, &mmr->mmr.mtt, mmr->umem);
	if (err) {
	mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr);
	ib_umem_release(mmr->umem);
	goto release_mpt_entry;
	}
	}

	/* If we couldn't transfer the MR to the HCA, just remember to
	* return a failure. But dereg_mr will free the resources.
	*/
	err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry);
	if (!err && flags & IB_MR_REREG_ACCESS)
	mmr->mmr.access = mr_access_flags;

	release_mpt_entry:
	mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry);

	return err;
	}

	int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
	{
	struct mlx4_ib_mr *mr = to_mmr(ibmr);
	int ret;

	ret = mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr);
	if (ret)
	return ret;
	if (mr->umem)
	ib_umem_release(mr->umem);
	kfree(mr);

	return 0;
	}

	struct ib_mw mlx4_ib_alloc_mw(struct ib_pd pd, enum ib_mw_type type)
	{
	struct mlx4_ib_dev *dev = to_mdev(pd->device);
	struct mlx4_ib_mw *mw;
	int err;

	mw = kmalloc(sizeof(*mw), GFP_KERNEL);
	if (!mw)
	return ERR_PTR(-ENOMEM);

	err = mlx4_mw_alloc(dev->dev, to_mpd(pd)->pdn,
	to_mlx4_type(type), &mw->mmw);
	if (err)
	goto err_free;

	err = mlx4_mw_enable(dev->dev, &mw->mmw);
	if (err)
	goto err_mw;

	mw->ibmw.rkey = mw->mmw.key;

	return &mw->ibmw;

	err_mw:
	mlx4_mw_free(dev->dev, &mw->mmw);

	err_free:
	kfree(mw);

	return ERR_PTR(err);
	}

	int mlx4_ib_bind_mw(struct ib_qp qp, struct ib_mw mw,
	struct ib_mw_bind *mw_bind)
	{
	struct ib_send_wr wr;
	struct ib_send_wr *bad_wr;
	int ret;

	memset(&wr, 0, sizeof(wr));
	wr.opcode = IB_WR_BIND_MW;
	wr.wr_id = mw_bind->wr_id;
	wr.send_flags = mw_bind->send_flags;
	wr.wr.bind_mw.mw = mw;
	wr.wr.bind_mw.bind_info = mw_bind->bind_info;
	wr.wr.bind_mw.rkey = ib_inc_rkey(mw->rkey);

	ret = mlx4_ib_post_send(qp, &wr, &bad_wr);
	if (!ret)
	mw->rkey = wr.wr.bind_mw.rkey;

	return ret;
	}

	int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)
	{
	struct mlx4_ib_mw *mw = to_mmw(ibmw);

	mlx4_mw_free(to_mdev(ibmw->device)->dev, &mw->mmw);
	kfree(mw);

	return 0;
	}

	struct ib_mr mlx4_ib_alloc_fast_reg_mr(struct ib_pd pd,
	int max_page_list_len)
	{
	struct mlx4_ib_dev *dev = to_mdev(pd->device);
	struct mlx4_ib_mr *mr;
	int err;

	mr = kmalloc(sizeof *mr, GFP_KERNEL);
	if (!mr)
	return ERR_PTR(-ENOMEM);

	err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0,
	max_page_list_len, 0, &mr->mmr);
	if (err)
	goto err_free;

	err = mlx4_mr_enable(dev->dev, &mr->mmr);
	if (err)
	goto err_mr;

	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
	mr->umem = NULL;

	return &mr->ibmr;

	err_mr:
	(void) mlx4_mr_free(dev->dev, &mr->mmr);

	err_free:
	kfree(mr);
	return ERR_PTR(err);
	}

	struct ib_fast_reg_page_list mlx4_ib_alloc_fast_reg_page_list(struct ib_device ibdev,
	int page_list_len)
	{
	struct mlx4_ib_dev *dev = to_mdev(ibdev);
	struct mlx4_ib_fast_reg_page_list *mfrpl;
	int size = page_list_len * sizeof (u64);

	if (page_list_len > MLX4_MAX_FAST_REG_PAGES)
	return ERR_PTR(-EINVAL);

	mfrpl = kmalloc(sizeof *mfrpl, GFP_KERNEL);
	if (!mfrpl)
	return ERR_PTR(-ENOMEM);

	mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL);
	if (!mfrpl->ibfrpl.page_list)
	goto err_free;

	mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev,
	size, &mfrpl->map,
	GFP_KERNEL);
	if (!mfrpl->mapped_page_list)
	goto err_free;

	WARN_ON(mfrpl->map & 0x3f);

	return &mfrpl->ibfrpl;

	err_free:
	kfree(mfrpl->ibfrpl.page_list);
	kfree(mfrpl);
	return ERR_PTR(-ENOMEM);
	}

	void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
	{
	struct mlx4_ib_dev *dev = to_mdev(page_list->device);
	struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list);
	int size = page_list->max_page_list_len * sizeof (u64);

	dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list,
	mfrpl->map);
	kfree(mfrpl->ibfrpl.page_list);
	kfree(mfrpl);
	}

	struct ib_fmr mlx4_ib_fmr_alloc(struct ib_pd pd, int acc,
	struct ib_fmr_attr *fmr_attr)
	{
	struct mlx4_ib_dev *dev = to_mdev(pd->device);
	struct mlx4_ib_fmr *fmr;
	int err = -ENOMEM;

	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
	if (!fmr)
	return ERR_PTR(-ENOMEM);

	err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc),
	fmr_attr->max_pages, fmr_attr->max_maps,
	fmr_attr->page_shift, &fmr->mfmr);
	if (err)
	goto err_free;

	err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr);
	if (err)
	goto err_mr;

	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key;

	return &fmr->ibfmr;

	err_mr:
	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr);

	err_free:
	kfree(fmr);

	return ERR_PTR(err);
	}

	int mlx4_ib_map_phys_fmr(struct ib_fmr ibfmr, u64 page_list,
	int npages, u64 iova)
	{
	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
	struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);

	return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,
	&ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
	}

	int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
	{
	struct ib_fmr *ibfmr;
	int err;
	struct mlx4_dev *mdev = NULL;

	list_for_each_entry(ibfmr, fmr_list, list) {
	if (mdev && to_mdev(ibfmr->device)->dev != mdev)
	return -EINVAL;
	mdev = to_mdev(ibfmr->device)->dev;
	}

	if (!mdev)
	return 0;

	list_for_each_entry(ibfmr, fmr_list, list) {
	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);

	mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
	}

	/*
	* Make sure all MPT status updates are visible before issuing
	* SYNC_TPT firmware command.
	*/
	wmb();

	err = mlx4_SYNC_TPT(mdev);
	if (err)
	pr_warn("SYNC_TPT error %d when "
	"unmapping FMRs\n", err);

	return 0;
	}

	int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)
	{
	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
	struct mlx4_ib_dev *dev = to_mdev(ibfmr->device);
	int err;

	err = mlx4_fmr_free(dev->dev, &ifmr->mfmr);

	if (!err)
	kfree(ifmr);

	return err;
	}