| /******************************************************************************* |
| Copyright (C) Marvell International Ltd. and its affiliates |
| |
| This software file (the "File") is owned and distributed by Marvell |
| International Ltd. and/or its affiliates ("Marvell") under the following |
| alternative licensing terms. Once you have made an election to distribute the |
| File under one of the following license alternatives, please (i) delete this |
| introductory statement regarding license alternatives, (ii) delete the two |
| license alternatives that you have not elected to use and (iii) preserve the |
| Marvell copyright notice above. |
| |
| |
| ******************************************************************************** |
| Marvell GPL License Option |
| |
| If you received this File from Marvell, you may opt to use, redistribute and/or |
| modify this File in accordance with the terms and conditions of the General |
| Public License Version 2, June 1991 (the "GPL License"), a copy of which is |
| available along with the File in the license.txt file or by writing to the Free |
| Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 or |
| on the worldwide web at http://www.gnu.org/licenses/gpl.txt. |
| |
| THE FILE IS DISTRIBUTED AS-IS, WITHOUT WARRANTY OF ANY KIND, AND THE IMPLIED |
| WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE ARE EXPRESSLY |
| DISCLAIMED. The GPL License provides additional details about this warranty |
| disclaimer. |
| *******************************************************************************/ |
| |
| #include <linux/types.h> |
| #include <linux/kernel.h> |
| #include <linux/init.h> |
| #include <linux/device.h> |
| #include <linux/string.h> |
| #include <linux/sysdev.h> |
| #include <linux/version.h> |
| #include <asm/uaccess.h> |
| #include <linux/proc_fs.h> |
| #include <linux/dmaengine.h> |
| |
| #include "mvCommon.h" |
| #include "mvOs.h" |
| #include "xor/mvXor.h" |
| #include "xor/mvXorRegs.h" |
| #include "mvSysXorApi.h" |
| #include "ctrlEnv/mvCtrlEnvLib.h" |
| |
| #define DUBUG |
| #undef DEBUG |
| #ifdef DEBUG |
| #define DPRINTK(s, args...) printk("xor: " s, ## args) |
| #else |
| #define DPRINTK(s, args...) |
| #endif |
| |
| /* |
| * Perform integrity test. |
| * Upon completion of xor dma operation, |
| * each descriptor will be checked to success bit. |
| */ |
| #define XOR_INTEGRITY |
| #undef XOR_INTEGRITY |
| |
| /* |
| * Double invalidate WA |
| */ |
| #ifdef CONFIG_MV_SP_I_FTCH_DB_INV |
| #define XOR_UNMAP |
| #endif |
| |
| /* |
| * Perform benchmark |
| */ |
| #define XOR_BENCH |
| #undef XOR_BENCH |
| |
| |
| #define XOR_STATS |
| //#undef XOR_STATS |
| #ifdef XOR_STATS |
| #define STAT_INC(s) xor_stats->s++ |
| #define STAT_ADD(s,n) xor_stats->s[((n)>>STAT_SHIFT)&(STAT_BYTES-1)]++ |
| #define STAT_SHIFT 7 |
| #define STAT_BYTES (1 << 9) |
| #else |
| #define STAT_INC(s) |
| #define STAT_ADD(s,n) |
| #endif |
| #if defined (CONFIG_MV_XOR_MEMCOPY) || defined (CONFIG_MV_IDMA_MEMCOPY) |
| #define memcpy asm_memcpy |
| #endif |
| |
| #define XOR_BUG(c) if (c) {xor_dump(); BUG();} |
| |
| #define CHANNELS CONFIG_MV_XOR_CHANNELS |
| #define ENGINES (CHANNELS >> 1) |
| #define XOR_TIMEOUT 0x8000000 |
| #define XOR_MIN_COPY_CHUNK 16 /* A0 specification */ |
| #define NET_DESC 128 /* NET DMA descriptors */ |
| #define DMA_DESC 2 /* memcpy descriptors */ |
| |
| #define to_dma_channel(dch) container_of(dch, struct xor_dma_channel, common) |
| |
| enum { |
| CHAIN_NETDMA = CHANNELS, |
| #ifdef CONFIG_MV_XOR_MEMCOPY |
| CHAIN_MEMCPY, |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMXOR |
| CHAIN_MEMXOR, |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_TO_USER |
| CHAIN_TO_USR, |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_FROM_USER |
| CHAIN_FROM_USR, |
| #endif |
| CHAINS, |
| }; |
| |
| typedef MV_XOR_DESC xor_desc_t; |
| |
| struct xor_chain { |
| unsigned int idx; |
| unsigned int pending; |
| xor_desc_t* desc; /* N descriptors */ |
| dma_addr_t base; /* phy address */ |
| struct xor_channel* owner; /* owned by channel */ |
| unsigned int busy; /* busy by cpu */ |
| unsigned int cookie_used; /* last used */ |
| unsigned int cookie_done; /* last completed */ |
| }; |
| |
| struct xor_channel { |
| unsigned int idx; |
| unsigned int busy; |
| struct xor_chain* chain; /* busy on chain */ |
| unsigned int pad; |
| }; |
| |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| struct xor_dma_device { |
| struct dma_device common; |
| }; |
| |
| struct xor_dma_channel { |
| struct dma_chan common; |
| struct xor_chain* chain; |
| spinlock_t lock; |
| }; |
| #endif |
| |
| struct xor_net_stats |
| { |
| #ifdef XOR_STATS |
| unsigned int err_chann_busy; |
| unsigned int err_chain_busy; |
| unsigned int err_addr; |
| unsigned int err_dma; |
| |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| unsigned int netdma_b2p; |
| unsigned int netdma_p2p; |
| unsigned int netdma_b2b; |
| unsigned int netdma_cpu; |
| unsigned int netdma_dma; |
| unsigned int netdma_complete; |
| unsigned int netdma_pending; |
| unsigned int netdma_bytes[STAT_BYTES]; |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMCOPY |
| unsigned int memcp; |
| unsigned int memcp_dma; |
| unsigned int memcp_bytes[STAT_BYTES]; |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMXOR |
| unsigned int memxor; |
| unsigned int memxor_dma; |
| unsigned int memxor_bytes[STAT_BYTES]; |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_TO_USER |
| unsigned int to_usr; |
| unsigned int to_usr_dma; |
| unsigned int to_usr_bytes[STAT_BYTES]; |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_FROM_USER |
| unsigned int from_usr; |
| unsigned int from_usr_dma; |
| unsigned int from_usr_bytes[STAT_BYTES]; |
| #endif |
| #endif |
| }; |
| |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| static struct xor_dma_device xor_dma_dev; |
| static struct xor_dma_channel xor_dma_chn[1]; |
| #endif |
| |
| struct xor_channel* xor_channels; |
| struct xor_chain* xor_chains; |
| struct xor_net_stats* xor_stats; |
| |
| static unsigned int xor_enabled __read_mostly; |
| |
| |
| |
| /* |
| * channel busy test |
| */ |
| static inline unsigned int xor_is_active(unsigned int i) |
| { |
| return MV_REG_READ(XOR_ACTIVATION_REG(XOR_UNIT(i),XOR_CHAN(i))) |
| /* & XEXACTR_XESTATUS_MASK */; |
| } |
| |
| /* |
| * set dma operation mode for channel i |
| */ |
| static inline void xor_mode_dma(unsigned int i) |
| { |
| unsigned int mode = MV_REG_READ(XOR_CONFIG_REG(XOR_UNIT(i),XOR_CHAN(i))); |
| mode &= ~XEXCR_OPERATION_MODE_MASK; |
| mode |= XEXCR_OPERATION_MODE_DMA; |
| MV_REG_WRITE(XOR_CONFIG_REG(XOR_UNIT(i), XOR_CHAN(i)), mode); |
| } |
| |
| /* |
| * set xor operation mode for channel i |
| */ |
| static inline void xor_mode_xor(unsigned int i) |
| { |
| unsigned int mode = MV_REG_READ(XOR_CONFIG_REG(XOR_UNIT(i),XOR_CHAN(i))); |
| mode &= ~XEXCR_OPERATION_MODE_MASK; |
| mode |= XEXCR_OPERATION_MODE_XOR; |
| MV_REG_WRITE(XOR_CONFIG_REG(XOR_UNIT(i), XOR_CHAN(i)), mode); |
| } |
| |
| /* |
| * run dma operation on channel |
| */ |
| static inline void xor_dma(unsigned int i, unsigned int base) |
| { |
| #ifdef MV_BRIDGE_SYNC_REORDER |
| mvOsBridgeReorderWA(); |
| #endif |
| MV_REG_WRITE(XOR_NEXT_DESC_PTR_REG(XOR_UNIT(i), XOR_CHAN(i)), base); |
| MV_REG_WRITE(XOR_ACTIVATION_REG(XOR_UNIT(i), XOR_CHAN(i)), XEXACTR_XESTART_MASK); |
| } |
| |
| /* |
| * xor engine busy wait |
| */ |
| static void xor_dump(void); |
| #define XOR_CAUSE_DONE_MASK(chan) ((BIT0|BIT1) << (chan * 16)) |
| void xor_wait(unsigned int chan) |
| { |
| unsigned int timeout; |
| unsigned int addr = XOR_CAUSE_REG(XOR_UNIT(chan)); |
| unsigned int mask = XOR_CAUSE_DONE_MASK(XOR_CHAN(chan)); |
| |
| timeout = XOR_TIMEOUT; |
| while(!(MV_REG_READ(addr) & mask)) |
| XOR_BUG(!timeout--); |
| |
| MV_REG_WRITE(addr, ~mask); |
| |
| timeout = XOR_TIMEOUT; |
| while(xor_is_active(chan)) |
| XOR_BUG(!timeout--); |
| } |
| |
| /* |
| * Second L2 invalidate (WA) |
| */ |
| #ifdef XOR_UNMAP |
| static void xor_chain_unmap(struct xor_chain* chain) |
| { |
| u32 va; |
| u32 i = chain->pending; |
| xor_desc_t* desc = chain->desc; |
| unsigned long flags; |
| |
| raw_local_irq_save(flags); |
| |
| while(i--) { |
| va = __phys_to_virt(desc->phyDestAdd); |
| /* invalidate l2 */ |
| __asm__ __volatile__ ("mcr p15, 1, %0, c15, c11, 4" : : "r" (va)); |
| __asm__ __volatile__ ("mcr p15, 1, %0, c15, c11, 5" : : "r" (va+desc->byteCnt-1)); |
| desc++; |
| } |
| |
| raw_local_irq_restore(flags); |
| } |
| #endif |
| /* |
| * Chain integrity test |
| */ |
| #ifdef XOR_INTEGRITY |
| static void xor_integrity(struct xor_chain* chain) |
| { |
| unsigned int i; |
| |
| for (i=0; i<chain->pending; i++) { |
| mvOsCacheLineInv(0, chain->desc + i); |
| if (chain->desc[i].status != 0x40000000) { |
| printk("XOR: violation chain[%d] desc=%d status=%x active=%d %x<-%x %d bytes\n", |
| chain->idx, i, |
| chain->desc[i].status, |
| xor_is_active(chain->idx), |
| chain->desc[i].phyDestAdd, chain->desc[i].srcAdd0, chain->desc[i].byteCnt); |
| } |
| } |
| } |
| #endif |
| |
| static void xor_dump(void) |
| { |
| unsigned int i; |
| |
| for (i=0; i<CHANNELS; i++) { |
| printk(" CHANNEL_ARBITER_REG %08x\n", MV_REG_READ(XOR_CHANNEL_ARBITER_REG(XOR_UNIT(i)))); |
| printk(" CONFIG_REG %08x\n", MV_REG_READ(XOR_CONFIG_REG(XOR_UNIT(i),XOR_CHAN(i)))); |
| printk(" ACTIVATION_REG %08x\n", MV_REG_READ(XOR_ACTIVATION_REG(XOR_UNIT(i), XOR_CHAN(i)))); |
| printk(" CAUSE_REG %08x\n", MV_REG_READ(XOR_CAUSE_REG(XOR_UNIT(i)))); |
| printk(" MASK_REG %08x\n", MV_REG_READ(XOR_MASK_REG(XOR_UNIT(i)))); |
| printk(" ERROR_CAUSE_REG %08x\n", MV_REG_READ(XOR_ERROR_CAUSE_REG(XOR_UNIT(i)))); |
| printk(" ERROR_ADDR_REG %08x\n", MV_REG_READ(XOR_ERROR_ADDR_REG(XOR_UNIT(i)))); |
| printk(" NEXT_DESC_PTR_REG %08x\n", MV_REG_READ(XOR_NEXT_DESC_PTR_REG(XOR_UNIT(i),XOR_CHAN(i)))); |
| printk(" CURR_DESC_PTR_REG %08x\n", MV_REG_READ(XOR_CURR_DESC_PTR_REG(XOR_UNIT(i),XOR_CHAN(i)))); |
| printk(" BYTE_COUNT_REG %08x\n\n", MV_REG_READ(XOR_BYTE_COUNT_REG(XOR_UNIT(i),XOR_CHAN(i)))); |
| } |
| |
| for (i=0; i<CHANNELS; i++) { |
| printk("channel[%d] active=0x%x busy=%d chain=%x\n", |
| xor_channels[i].idx, |
| xor_is_active(i), |
| xor_channels[i].busy, |
| xor_channels[i].chain ? xor_channels[i].chain->idx : ~0); |
| } |
| |
| for (i=0; i<CHAINS; i++) { |
| printk("chain[%d] on channel=%8x desc=%x cookies(%x,%x) pending=%d\n", i, |
| xor_chains[i].owner ? xor_chains[i].owner->idx : ~0, |
| xor_chains[i].base, |
| xor_chains[i].cookie_done, |
| xor_chains[i].cookie_used, |
| xor_chains[i].pending); |
| } |
| } |
| |
| static void xor_stat(void) |
| { |
| #ifdef XOR_STATS |
| int i; |
| |
| printk("XOR errors...........%10u\n", xor_stats->err_dma); |
| printk("XOR busy channel.....%10u\n", xor_stats->err_chann_busy); |
| printk("XOR busy chain.......%10u\n", xor_stats->err_chain_busy); |
| printk("XOR invalid address..%10u\n", xor_stats->err_addr); |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| printk("\n"); |
| printk("NETDMA b2p...........%10u\n", xor_stats->netdma_b2p); |
| printk("NETDMA p2p...........%10u\n", xor_stats->netdma_p2p); |
| printk("NETDMA b2b...........%10u\n", xor_stats->netdma_b2b); |
| printk("NETDMA by cpu........%10u\n", xor_stats->netdma_cpu); |
| printk("NETDMA by hw.........%10u\n", xor_stats->netdma_dma); |
| printk("NETDMA pending.......%10u\n", xor_stats->netdma_pending); |
| printk("NETDMA complete......%10u\n", xor_stats->netdma_complete); |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMCOPY |
| printk("\n"); |
| printk("MEMCPY total.........%10u\n", xor_stats->memcp); |
| printk("MEMCPY by hw.........%10u\n", xor_stats->memcp_dma); |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMXOR |
| printk("\n"); |
| printk("MEMXOR total.........%10u\n", xor_stats->memxor); |
| printk("MEMXOR by hw.........%10u\n", xor_stats->memxor_dma); |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_TO_USER |
| printk("\n"); |
| printk("TO_USER total.........%10u\n", xor_stats->to_usr); |
| printk("TO_USER by hw.........%10u\n", xor_stats->to_usr_dma); |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_FROM_USER |
| printk("\n"); |
| printk("FROM_USER total.........%10u\n", xor_stats->from_usr); |
| printk("FROM_USER by hw.........%10u\n", xor_stats->from_usr_dma); |
| #endif |
| |
| printk("BYTES NETDMA MEMCPY MEMXOR TO_USER FROM_USER\n"); |
| for (i=0; i<STAT_BYTES; i++) { |
| u32 a=0, b=0, c=0, d=0, e=0; |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| a = xor_stats->netdma_bytes[i]; |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMCOPY |
| b = xor_stats->memcp_bytes[i]; |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMXOR |
| c = xor_stats->memxor_bytes[i]; |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_TO_USER |
| d = xor_stats->to_usr_bytes[i]; |
| #endif |
| #ifdef CONFIG_MV_XOR_COPY_FROM_USER |
| e = xor_stats->from_usr_bytes[i]; |
| #endif |
| if (a || b || c || d || e) |
| printk("%5d%8u %8u %8u %8u %8u\n", i<<STAT_SHIFT, a, b, c, d, e); |
| } |
| |
| memset(xor_stats, 0, sizeof(struct xor_net_stats)); |
| #endif |
| } |
| |
| /* |
| * Allocate chain |
| */ |
| static inline unsigned int xor_try_chain(struct xor_chain* chain) |
| { |
| unsigned long flags=0; |
| local_irq_save(flags); |
| if (chain->busy) { |
| local_irq_restore(flags); |
| STAT_INC(err_chain_busy); |
| return 1; |
| } |
| chain->busy = 1; |
| local_irq_restore(flags); |
| return 0; |
| } |
| |
| /* |
| * Connect channel to chain |
| */ |
| static inline void xor_attach(struct xor_chain* chain, struct xor_channel* xch) |
| { |
| xch->chain = chain; |
| chain->owner = xch; |
| } |
| |
| /* |
| * Disconnect channel and chain |
| */ |
| static inline void xor_detach(struct xor_chain* chain, struct xor_channel* xch) |
| { |
| #ifdef XOR_INTEGRITY |
| xor_integrity(chain); |
| #endif |
| #ifdef XOR_UNMAP |
| if (chain->pending) |
| xor_chain_unmap(chain); |
| #endif |
| chain->cookie_done = chain->cookie_used; |
| chain->pending = 0; |
| chain->owner = NULL; |
| chain->busy = 0; |
| xch->chain = NULL; |
| } |
| |
| /* |
| * Allocate channel |
| */ |
| static struct xor_channel* xor_get(void) |
| { |
| static int rr=0; |
| int retry=CHANNELS; |
| struct xor_channel* xch = NULL; |
| unsigned long flags=0; |
| |
| local_irq_save(flags); |
| |
| rr += ENGINES; |
| |
| while(retry--) { |
| rr &= (CHANNELS-1); |
| |
| if (xor_channels[rr].busy || xor_is_active(rr)) { |
| rr++; |
| continue; |
| } |
| |
| xch = &xor_channels[rr]; |
| xch->busy = 1; |
| |
| if (xch->chain) |
| xor_detach(xch->chain, xch); |
| |
| break; |
| } |
| |
| local_irq_restore(flags); |
| |
| if (!xch) |
| STAT_INC(err_chann_busy); |
| |
| return xch; |
| } |
| |
| static inline void xor_put(struct xor_channel* xch) |
| { |
| xch->busy = 0; |
| } |
| |
| /* |
| * Fast clean, dsb() required |
| * The call is intended for multiple calls and when dsb() on commit endpoint |
| */ |
| static inline void dmac_clean_dcache_line(void* addr) |
| { |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 1" : : "r" (addr)); |
| __asm__ __volatile__ ("mcr p15, 1, %0, c15, c9, 1" : : "r" (addr)); |
| |
| #else |
| dma_cache_maint(addr, 32, DMA_TO_DEVICE); |
| #endif |
| } |
| |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| static int xor_alloc_chan_resources(struct dma_chan *dch) |
| { |
| return 1; |
| } |
| |
| static void xor_free_chan_resources(struct dma_chan *dch) |
| { |
| printk("xor: free netdma chain[%d] resources\n", dch->chan_id); |
| } |
| |
| /* |
| * Start xmit of chain |
| */ |
| static struct xor_chain* xor_xmit(struct xor_chain* chain) |
| { |
| struct xor_channel* xch; |
| int i; |
| |
| xch = xor_get(); |
| if (!xch) |
| goto out; |
| |
| xor_attach(chain, xch); |
| |
| i = chain->pending; |
| while(i--) |
| dmac_clean_dcache_line(chain->desc + i); |
| dsb(); |
| |
| STAT_INC(netdma_dma); |
| xor_dma(xch->idx, chain->base); |
| xor_put(xch); |
| |
| /* |
| * find free chain, |
| * always succedded since chains >> channels |
| */ |
| i = CHAIN_NETDMA; |
| while(i--) { |
| if (likely(chain->idx < CHAIN_NETDMA)) |
| chain++; |
| else |
| chain = xor_chains; |
| |
| if (chain->owner) |
| continue; |
| |
| if (chain->busy) |
| continue; |
| |
| chain->busy = 1; |
| break; |
| } |
| |
| out: |
| return chain; |
| } |
| |
| static dma_cookie_t xor_memcpy_buf_to_pg(struct dma_chan *chan, |
| struct page *page, unsigned int offset, void *_from, size_t n) |
| { |
| u32 to, from=(u32)_from; |
| xor_desc_t* desc; |
| struct xor_dma_channel* dch = to_dma_channel(chan); |
| struct xor_chain* chain; |
| int irq = in_interrupt(); |
| u32 c; |
| |
| STAT_INC(netdma_b2p); |
| STAT_ADD(netdma_bytes, n); |
| |
| to = (u32)page_address(page) + offset; |
| |
| DPRINTK("%s: %p<-%p %d bytes\n", __FUNCTION__, to, from, n); |
| |
| if (!virt_addr_valid(to) || !virt_addr_valid(from)) { |
| STAT_INC(err_addr); |
| goto out; |
| } |
| |
| /* at least two cache lines or XOR minimum */ |
| if (n < 96) |
| goto out; |
| |
| if (to & 31) { |
| c = 32 - (to & 31); |
| memcpy((void*)to, (void*)from, c); |
| to += c; |
| from += c; |
| n -= c; |
| } |
| |
| c = (to+n) & 31; |
| if (c) { |
| n -= c; |
| memcpy((void*)to+n, (void*)from+n, c); |
| } |
| |
| if (n < 32) |
| goto out; |
| |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| dmac_clean_range((void*)from, (void*)from + n); |
| dmac_inv_range((void*)to, (void*)to + n); |
| #else |
| dma_cache_maint((void*)from, n, DMA_TO_DEVICE); |
| dma_cache_maint((void*)to, n, DMA_FROM_DEVICE); |
| #endif |
| |
| if (!irq) |
| spin_lock_bh(&dch->lock); |
| |
| chain = dch->chain; |
| |
| XOR_BUG(chain->owner || chain->pending>=NET_DESC); |
| |
| desc = chain->desc + chain->pending++; |
| desc->srcAdd0 = __pa((void*)from); |
| desc->phyDestAdd = __pa((void*)to); |
| desc->byteCnt = n; |
| desc->status = BIT31; |
| |
| if (chain->pending == 1) |
| chain->cookie_done = chan->cookie; |
| |
| if (++chan->cookie < 0) |
| chan->cookie = 1; |
| |
| chain->cookie_used = chan->cookie; |
| |
| if (chain->pending > 1) |
| dch->chain = xor_xmit(chain); |
| |
| if (!irq) |
| spin_unlock_bh(&dch->lock); |
| |
| return chan->cookie; |
| |
| out: |
| STAT_INC(netdma_cpu); |
| memcpy((void*)to, (void*)from, n); |
| return chan->cookie; |
| } |
| |
| static dma_cookie_t xor_memcpy_pg_to_pg(struct dma_chan *chan, |
| struct page *dest_pg, unsigned int dest_off, |
| struct page *src_pg, unsigned int src_off, size_t n) |
| { |
| STAT_INC(netdma_p2p); |
| return xor_memcpy_buf_to_pg(chan, |
| dest_pg, |
| dest_off, |
| (void*)((u32)page_address(src_pg) + src_off), n); |
| } |
| |
| static dma_cookie_t xor_memcpy_buf_to_buf(struct dma_chan *chan,void *dest, void *src, size_t n) |
| { |
| /* kernel copies (e.g. smbfs) |
| * not really tested |
| */ |
| STAT_INC(netdma_b2b); |
| return xor_memcpy_buf_to_pg(chan, |
| page_address(dest), |
| page_offset(dest), |
| src, n); |
| } |
| |
| /* |
| * cookies compare |
| */ |
| static inline int before(__s32 c1, __s32 c2) |
| { |
| /* |
| * Cookies are continuous. |
| * The distance between cookies should not exceed FFFF, |
| * otherwise is considered as overflow. |
| */ |
| if (c1 <= c2) |
| return (c2 - c1) <= 0xFFFF; |
| else |
| return (c1 - c2) > 0xFFFF; |
| } |
| |
| /* |
| * Find completed cookie across all chains. |
| * Each one has [done,used] section. If for sections done=used, report max, |
| * otherwise, the minimum 'done' is returned. |
| */ |
| static /*inline*/ dma_cookie_t xor_get_completed(void) |
| { |
| dma_cookie_t min_cookie = 0; |
| dma_cookie_t max_cookie = 0; |
| dma_cookie_t c_used, c_done; |
| |
| struct xor_chain* chain = xor_chains; |
| int i = CHAIN_NETDMA; |
| |
| do { |
| c_used = chain->cookie_used; |
| c_done = chain->cookie_done; |
| |
| if (c_used != c_done) { |
| if (chain->owner && !xor_is_active(chain->owner->idx)) { |
| c_used = c_done = chain->cookie_used; |
| } |
| } |
| |
| if (c_used != c_done) { |
| /* |
| * at least one chain is incomplete. |
| * find minimal cookie which is done across all chains |
| */ |
| if (!min_cookie || before(c_done, min_cookie)) |
| min_cookie = c_done; |
| } |
| |
| /* |
| * in the case whether all chains are completed, |
| * find their max cookie |
| */ |
| else if (!c_done) { |
| ; |
| } |
| else if (!max_cookie || before(max_cookie, c_done)) |
| max_cookie = c_done; |
| |
| chain++; |
| |
| } while(i--); |
| |
| return min_cookie ? min_cookie: max_cookie; |
| } |
| #ifdef XOR_UNMAP |
| static void xor_memcpy_cleanup(struct dma_chan *chan, dma_cookie_t cookie) |
| { |
| struct xor_chain* chain = xor_chains; |
| struct xor_dma_channel* dch = to_dma_channel(chan); |
| int i = CHAIN_NETDMA; |
| |
| spin_lock_bh(&dch->lock); |
| |
| do { |
| if (chain->owner && |
| chain->pending && !xor_is_active(chain->owner->idx)) { |
| xor_detach(chain, chain->owner); |
| } |
| |
| chain++; |
| |
| } while(i--); |
| |
| spin_unlock_bh(&dch->lock); |
| } |
| #endif |
| |
| static enum dma_status xor_memcpy_complete(struct dma_chan *chan, |
| dma_cookie_t cookie, |
| dma_cookie_t *pdone, |
| dma_cookie_t *pused) |
| { |
| dma_cookie_t done; |
| dma_cookie_t used; |
| enum dma_status ret; |
| |
| STAT_INC(netdma_complete); |
| |
| done = xor_get_completed(); |
| used = chan->cookie; |
| |
| if (pdone) |
| *pdone = done; |
| if (pused) |
| *pused = used; |
| |
| ret = dma_async_is_complete(cookie, done, used); |
| #ifdef XOR_UNMAP |
| if (ret == DMA_SUCCESS) |
| xor_memcpy_cleanup(chan, cookie); |
| #endif |
| |
| return ret; |
| } |
| |
| static void xor_memcpy_issue_pending(struct dma_chan *chan) |
| { |
| struct xor_dma_channel* dch = to_dma_channel(chan); |
| struct xor_chain* chain; |
| unsigned int irq = in_interrupt(); |
| |
| STAT_INC(netdma_pending); |
| |
| chain = dch->chain; |
| if (chain->owner) |
| return; |
| |
| if (!irq) |
| spin_lock_bh(&dch->lock); |
| |
| chain = dch->chain; |
| |
| if (!chain->owner && chain->pending) { |
| do { |
| dch->chain = xor_xmit(chain); |
| } while (dch->chain == chain); |
| } |
| |
| if (!irq) |
| spin_unlock_bh(&dch->lock); |
| } |
| #endif /* CONFIG_MV_XOR_NET_DMA */ |
| |
| static int xor_proc_write(struct file *file, const char __user *buffer, |
| unsigned long count, void *data) |
| { |
| #ifdef XOR_BENCH |
| xor_bench(NULL,NULL,0); |
| #endif |
| return count; |
| } |
| |
| static int xor_proc_read(char *page, char **start, off_t off, |
| int count, int *eof, void *data) |
| { |
| xor_dump(); |
| xor_stat(); |
| return 0; |
| } |
| |
| int __init mv_xor_init(void) |
| { |
| unsigned int i; |
| struct proc_dir_entry *proc_e; |
| struct xor_chain* chain; |
| u32 va; |
| |
| if ((MV_FALSE == mvCtrlPwrClckGet(XOR_UNIT_ID, 0)) || |
| (MV_FALSE == mvCtrlPwrClckGet(XOR_UNIT_ID, 1))) |
| { |
| printk("\nWarning at least one XOR port is Powered Off\n"); |
| return 0; |
| |
| } |
| |
| |
| #if defined(CONFIG_MV78200) || defined(CONFIG_MV632X) |
| if (MV_FALSE == mvSocUnitIsMappedToThisCpu(XOR)) { |
| printk(KERN_INFO"XOR engine is not mapped to this CPU\n"); |
| return -ENODEV; |
| } |
| #endif |
| |
| mvSysXorInit(); |
| |
| /* stats */ |
| va = (u32)kmalloc(32 + (sizeof(struct xor_net_stats)), GFP_KERNEL); |
| va += 32 - (va & 31); |
| xor_stats = (struct xor_net_stats*)va; |
| memset(xor_stats, 0, sizeof(struct xor_net_stats)); |
| |
| /* channels */ |
| va = (u32)kmalloc(32 + (sizeof(struct xor_channel) * CHANNELS), GFP_KERNEL); |
| va += 32 - (va & 31); |
| xor_channels = (struct xor_channel*)va; |
| memset(xor_channels, 0, sizeof(struct xor_channel) * CHANNELS); |
| |
| /* chains */ |
| va = (u32)kmalloc(32 + (sizeof(struct xor_chain) * CHAINS), GFP_KERNEL); |
| va += 32 - (va & 31); |
| xor_chains = (struct xor_chain*)va; |
| memset(xor_chains, 0, sizeof(struct xor_chain) * CHAINS); |
| |
| /* |
| * init channels |
| */ |
| for(i=0; i<CHANNELS; i++) { |
| xor_channels[i].idx = i; |
| xor_channels[i].busy = 0; |
| xor_channels[i].chain = NULL; |
| |
| xor_mode_dma(i); |
| } |
| |
| /* |
| * init chains |
| */ |
| i = CHAINS; |
| while(i--) { |
| u32 sz, va=0; |
| |
| chain = &xor_chains[i]; |
| |
| chain->idx = i; |
| |
| if (i <= CHAIN_NETDMA) |
| sz = sizeof(xor_desc_t) * NET_DESC; |
| else |
| sz = sizeof(xor_desc_t) * DMA_DESC; |
| |
| va = (u32)kmalloc(sz+64, GFP_KERNEL); |
| va += 64 - (va & 63); |
| chain->desc = (void*)va; |
| chain->base = __pa(va); |
| |
| memset(chain->desc, 0, sz); |
| BUG_ON(63 & (u32)chain->desc); |
| |
| /* |
| * Init next pointer |
| */ |
| if (i <= CHAIN_NETDMA) { |
| int d = NET_DESC - 1; |
| while(d--) |
| chain->desc[d].phyNextDescPtr = |
| chain->base + (sizeof(xor_desc_t) * (d+1)); |
| } |
| |
| } |
| |
| |
| #ifdef CONFIG_PROC_FS |
| /* FIXME: /proc/sys/dev/ */ |
| proc_e = create_proc_entry("mvxor", 0666, 0); |
| proc_e->read_proc = xor_proc_read; |
| proc_e->write_proc = xor_proc_write; |
| proc_e->nlink = 1; |
| #endif |
| |
| #ifdef CONFIG_MV_XOR_NET_DMA |
| memset(&xor_dma_dev, 0, sizeof(struct xor_dma_device)); |
| memset(xor_dma_chn, 0, sizeof(struct xor_dma_channel)); |
| INIT_LIST_HEAD(&xor_dma_dev.common.channels); |
| |
| xor_dma_dev.common.chancnt = 1; |
| xor_dma_chn[0].common.device = &xor_dma_dev.common; |
| xor_dma_chn[0].chain = &xor_chains[0]; |
| xor_dma_chn[0].chain->busy = 1; |
| |
| spin_lock_init(&xor_dma_chn[0].lock); |
| list_add_tail(&xor_dma_chn[0].common.device_node, |
| &xor_dma_dev.common.channels); |
| |
| xor_dma_dev.common.device_alloc_chan_resources = xor_alloc_chan_resources; |
| xor_dma_dev.common.device_free_chan_resources = xor_free_chan_resources; |
| xor_dma_dev.common.device_memcpy_buf_to_buf = xor_memcpy_buf_to_buf; |
| xor_dma_dev.common.device_memcpy_buf_to_pg = xor_memcpy_buf_to_pg; |
| xor_dma_dev.common.device_memcpy_pg_to_pg = xor_memcpy_pg_to_pg; |
| xor_dma_dev.common.device_memcpy_complete = xor_memcpy_complete; |
| xor_dma_dev.common.device_memcpy_issue_pending = xor_memcpy_issue_pending; |
| |
| if (dma_async_device_register(&xor_dma_dev.common)) { |
| printk(KERN_ERR" XOR engine cannot be registered as async NET_DMA\n"); |
| return -ENODEV; |
| } |
| |
| printk(KERN_INFO "XOR registered %d NET_DMA over %d channels\n", |
| xor_dma_dev.common.chancnt, CHANNELS); |
| |
| #else |
| printk(KERN_INFO "XOR registered %d channels\n", CHANNELS); |
| |
| #endif |
| |
| #ifdef XOR_UNMAP |
| printk(KERN_INFO "XOR 2nd invalidate WA enabled\n"); |
| #endif |
| xor_enabled = 1; |
| return 0; |
| } |
| |
| void __exit mv_xor_exit(void) |
| { |
| return; |
| } |
| |
| module_init(mv_xor_init); |
| module_exit(mv_xor_exit); |
| MODULE_LICENSE(GPL); |
| |
| |
| /* = ====================================================================== */ |
| #ifdef CONFIG_MV_XOR_MEMCOPY |
| void* xor_memcpy(void *_to, const void* _from, __kernel_size_t n) |
| { |
| u32 to = (u32)_to; |
| u32 from = (u32)_from; |
| u32 sa, da; |
| |
| struct xor_chain* chain; |
| struct xor_channel *xch; |
| |
| if (!xor_enabled) |
| return asm_memcpy((void*)to, (void*)from, n); |
| |
| STAT_INC(memcp); |
| STAT_ADD(memcp_bytes,n); |
| |
| DPRINTK("%s:%x<-%x %d bytes\n", __FUNCTION__, to, from, n); |
| |
| if (n < 128) |
| goto out; |
| |
| if (!virt_addr_valid(to) || !virt_addr_valid(from)) { |
| STAT_INC(err_addr); |
| goto out; |
| } |
| |
| da = __pa((void*)to); |
| sa = __pa((void*)from); |
| |
| if (((da < sa) && (sa < da+n)) || |
| ((sa < da) && (da < sa+n))) |
| return memmove((void*)to, (void*)from, n); |
| |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| dmac_clean_range((void*)from, (void*)from + n); |
| dmac_inv_range((void*)to, (void*)to + n); |
| #else |
| dma_cache_maint((void*)from, n, DMA_TO_DEVICE); |
| dma_cache_maint((void*)to, n, DMA_FROM_DEVICE); |
| #endif |
| |
| da = virt_to_phys((void*)to); |
| sa = virt_to_phys((void*)from); |
| |
| if (unlikely(!sa || !da)) { |
| STAT_INC(err_addr); |
| goto out; |
| } |
| |
| chain = &xor_chains[CHAIN_MEMCPY]; |
| if (xor_try_chain(chain)) |
| goto out; |
| |
| DPRINTK("%s: %x<-%x %d bytes\n", __FUNCTION__, da, sa, n); |
| chain->pending = 1; |
| chain->desc->srcAdd0 = sa; |
| chain->desc->phyDestAdd = da; |
| chain->desc->byteCnt = n; |
| chain->desc->phyNextDescPtr = 0; |
| chain->desc->status = BIT31; |
| |
| dmac_clean_dcache_line(chain->desc); |
| dsb(); |
| |
| xch = xor_get(); |
| if (!xch) |
| goto out2; |
| |
| STAT_INC(memcp_dma); |
| xor_attach(chain, xch); |
| xor_dma(xch->idx, chain->base); |
| |
| { |
| u32 c; |
| |
| da = to & 31; |
| sa = from & 31; |
| if (sa || da) { |
| c = (sa > da) ? 32 - da : 32 - sa; |
| memmove((void*)to, (void*)from, c); |
| } |
| |
| da = (to+n) & 31; |
| sa = (from+n) & 31; |
| if (sa || da) { |
| c = (da > sa) ? da : sa; |
| memmove((void*)(to+n-c), (void*)(from+n-c), c); |
| } |
| } |
| |
| xor_wait(xch->idx); |
| xor_detach(chain, xch); |
| xor_put(xch); |
| |
| return (void*)to; |
| |
| out2: |
| chain->busy = 0; |
| out: |
| return asm_memcpy((void*)to, (void*)from, n); |
| } |
| EXPORT_SYMBOL(xor_memcpy); |
| #endif /* CONFIG_MV_XOR_MEMCOPY */ |
| |
| #ifdef CONFIG_MV_XOR_MEMXOR |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| int xor_memxor(unsigned int count, unsigned int n, void **va) |
| { |
| struct xor_chain* chain; |
| struct xor_channel *xch; |
| u32* p; |
| |
| if (!xor_enabled) |
| goto out; |
| |
| BUG_ON(count <= 1); |
| |
| DPRINTK("%s: %d sources %d bytes\n", __FUNCTION__, count, n); |
| |
| chain = &xor_chains[CHAIN_MEMXOR]; |
| if (xor_try_chain(chain)) |
| goto out; |
| |
| chain->pending = 1; |
| chain->desc->phyDestAdd = __pa(va[0]); |
| chain->desc->byteCnt = n; |
| chain->desc->phyNextDescPtr = 0; |
| chain->desc->descCommand = (1 << count) - 1; |
| chain->desc->status = BIT31; |
| |
| p = &chain->desc->srcAdd0; |
| |
| #ifdef MV_CPU_BE |
| p = &chain->desc->srcAdd1; |
| if (count & 1) |
| p[count] = __pa(va[count-1]); |
| #endif |
| |
| while(count--) { |
| dmac_clean_range(va[count], (void*)va[count] + n); |
| p[count] = __pa(va[count]); |
| } |
| |
| dmac_clean_dcache_line(chain->desc); |
| dmac_clean_dcache_line(chain->desc+32); |
| dsb(); |
| |
| xch = xor_get(); |
| if (!xch) |
| goto out2; |
| |
| STAT_ADD(memxor_bytes, n); |
| STAT_INC(memxor_dma); |
| |
| xor_attach(chain, xch); |
| xor_mode_xor(xch->idx); |
| xor_dma(xch->idx, chain->base); |
| |
| /* could be useful before busywait, because we already have it clean */ |
| dmac_inv_range(va[0], (void*)va[0] + n); |
| xor_wait(xch->idx); |
| |
| xor_mode_dma(xch->idx); |
| xor_detach(chain, xch); |
| wmb(); |
| xor_put(xch); |
| |
| return 0; |
| |
| out2: |
| chain->busy = 0; |
| out: |
| return 1; |
| } |
| #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2,6,26) */ |
| int xor_memxor(unsigned int src_count, unsigned int bytes, void *dest, void **srcs) |
| { |
| struct xor_chain* chain; |
| struct xor_channel *xch; |
| u32* p; |
| |
| if (!xor_enabled) |
| goto out; |
| |
| BUG_ON(src_count < 1); |
| |
| DPRINTK("%s: %d sources %d bytes\n", __FUNCTION__, src_count, bytes); |
| |
| chain = &xor_chains[CHAIN_MEMXOR]; |
| if (xor_try_chain(chain)) |
| goto out; |
| |
| chain->pending = 1; |
| chain->desc->phyDestAdd = __pa(dest); |
| chain->desc->byteCnt = bytes; |
| chain->desc->phyNextDescPtr = 0; |
| chain->desc->descCommand = (1 << (src_count+1)) - 1; /* add dest to the count */ |
| chain->desc->status = BIT31; |
| |
| #ifdef MV_CPU_BE |
| /* set the first (srcAdd1) desc to be the dest */ |
| chain->desc->srcAdd1 = chain->desc->phyDestAdd; |
| p = &chain->desc->srcAdd0; |
| if ((src_count+1) & 1) |
| p[src_count] = __pa(srcs[src_count-1]); |
| #else |
| /* set the first desc (srcAdd0) to be the dest */ |
| chain->desc->srcAdd0 = chain->desc->phyDestAdd; |
| p = &chain->desc->srcAdd1; |
| #endif |
| |
| dma_cache_maint(dest, bytes, DMA_TO_DEVICE); |
| |
| while(src_count--) { |
| dma_cache_maint((void*)srcs[src_count], bytes, DMA_TO_DEVICE); |
| p[src_count] = __pa(srcs[src_count]); |
| } |
| |
| dmac_clean_dcache_line(chain->desc); |
| dmac_clean_dcache_line(chain->desc+32); |
| dsb(); |
| |
| xch = xor_get(); |
| if (!xch) |
| goto out2; |
| |
| STAT_ADD(memxor_bytes, bytes); |
| STAT_INC(memxor_dma); |
| |
| xor_attach(chain, xch); |
| xor_mode_xor(xch->idx); |
| xor_dma(xch->idx, chain->base); |
| |
| /* could be useful before busywait, because we already have it clean */ |
| dma_cache_maint(dest, bytes, DMA_FROM_DEVICE); |
| xor_wait(xch->idx); |
| |
| xor_mode_dma(xch->idx); |
| xor_detach(chain, xch); |
| wmb(); |
| xor_put(xch); |
| |
| return 0; |
| |
| out2: |
| chain->busy = 0; |
| out: |
| return 1; |
| } |
| |
| #endif |
| EXPORT_SYMBOL(xor_memxor); |
| #endif /* CONFIG_MV_XOR_MEMXOR */ |
| |
| |
| #if defined(CONFIG_MV_XOR_COPY_TO_USER) || defined(CONFIG_MV_XOR_COPY_FROM_USER) |
| /* |
| * Obtain pa address from user va |
| */ |
| static inline u32 __pa_user(u32 va, int write) |
| { |
| struct page *page; |
| struct vm_area_struct *vm; |
| struct mm_struct * mm = (va >= TASK_SIZE)? &init_mm : current->mm; |
| unsigned int flags; |
| |
| if (virt_addr_valid(va)) |
| return __pa(va); |
| |
| if (va >= (u32)high_memory) |
| return 0; |
| |
| flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); |
| |
| vm = find_extend_vma(mm, va); |
| if (!vm || (vm->vm_flags & (VM_IO|VM_PFNMAP)) || !(flags & vm->vm_flags)) |
| return 0; |
| |
| flags = FOLL_PTE_EXIST | FOLL_TOUCH; |
| flags |= (write) ? FOLL_WRITE : 0; |
| |
| page = follow_page(vm, va, flags); |
| |
| if (pfn_valid(page_to_pfn(page))) |
| return ((page_to_pfn(page) << PAGE_SHIFT) | (va & (PAGE_SIZE - 1))); |
| |
| return 0; |
| } |
| #endif /* COPY_XX_USER */ |
| |
| #ifdef CONFIG_MV_XOR_COPY_TO_USER |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| unsigned long xor_copy_to_user(unsigned long to, unsigned long from, unsigned long n) |
| { |
| u32 sa, da, copy; |
| struct xor_chain* chain; |
| struct xor_channel* xch; |
| unsigned int i = 0; |
| unsigned long rc; |
| unsigned long flags=0; |
| |
| if (unlikely(!xor_enabled)) |
| goto out; |
| |
| STAT_INC(to_usr); |
| STAT_ADD(to_usr_bytes, n); |
| |
| DPRINTK("%s: %p<-%p %d bytes\n", __FUNCTION__, to, from, n); |
| |
| if (!virt_addr_valid(from)) { |
| STAT_INC(err_addr); |
| goto out; |
| } |
| |
| if (to & 31) |
| mvOsCacheLineFlushInv(0, to); |
| |
| if ((to+n) & 31) |
| mvOsCacheLineFlushInv(0, to+n); |
| |
| dmac_clean_range((void*)from, (void*)from + n); |
| |
| chain = &xor_chains[CHAIN_TO_USR]; |
| if (xor_try_chain(chain)) |
| goto out; |
| |
| local_irq_save(flags); |
| |
| while(n) { |
| copy = PAGE_SIZE - (to & ~PAGE_MASK); |
| if (copy > n) |
| copy = n; |
| |
| if (!copy) |
| break; |
| |
| if (copy < XOR_MIN_COPY_CHUNK) { |
| rc= __arch_copy_to_user((void*)to, (void*)from, copy); |
| if (rc) |
| goto out2; |
| } |
| else { |
| |
| sa = __pa(from); |
| da = __pa_user(to, 1); |
| |
| if (unlikely(!sa || !da)) { |
| STAT_INC(err_addr); |
| goto out2; |
| } |
| |
| DPRINTK("%s: desc[%d] %x<-%x %d bytes\n", __FUNCTION__, i, da, sa, copy); |
| |
| dmac_inv_range((void*)to, (void*)to + copy); |
| |
| chain->desc[i].srcAdd0 = sa; |
| chain->desc[i].phyDestAdd = da; |
| chain->desc[i].byteCnt = copy; |
| chain->desc[i].phyNextDescPtr = 0; |
| chain->desc[i].status = BIT31; |
| if (i) |
| chain->desc[i-1].phyNextDescPtr = |
| chain->base + (sizeof(xor_desc_t) * i); |
| |
| i++; |
| } |
| |
| from += copy; |
| to += copy; |
| n -= copy; |
| } |
| |
| if (i) { |
| |
| while(i--) |
| dmac_clean_dcache_line(chain->desc + i); |
| dsb(); |
| |
| xch = xor_get(); |
| if (!xch) |
| goto out2; |
| |
| xor_attach(chain, xch); |
| |
| DPRINTK("%s: ch[%d] chain=%d\n", __FUNCTION__, xch->idx, chain->idx); |
| |
| STAT_INC(to_usr_dma); |
| xor_dma(xch->idx, chain->base); |
| xor_wait(xch->idx); |
| |
| xor_detach(chain, xch); |
| xor_put(xch); |
| } |
| |
| local_irq_restore(flags); |
| |
| return 0; |
| |
| out2: |
| chain->busy=0; |
| local_irq_restore(flags); |
| out: |
| return __arch_copy_to_user((void*)to, (void*)from, n); |
| } |
| EXPORT_SYMBOL(xor_copy_to_user); |
| #else |
| #error "Kernel version >= 2,6,26 does not support XOR copy_to_user" |
| #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) */ |
| #endif /* CONFIG_MV_XOR_COPY_TO_USER */ |
| |
| #ifdef CONFIG_MV_XOR_COPY_FROM_USER |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| unsigned long xor_copy_from_user(unsigned long to, unsigned long from, unsigned long n) |
| { |
| u32 sa, da, copy; |
| struct xor_chain* chain; |
| struct xor_channel* xch; |
| unsigned int i = 0; |
| unsigned long flags=0; |
| unsigned long rc; |
| |
| if (unlikely(!xor_enabled)) |
| goto out; |
| |
| STAT_INC(from_usr); |
| STAT_ADD(from_usr_bytes, n); |
| |
| DPRINTK("%s: %p<-%p %d bytes\n", __FUNCTION__, to, from, n); |
| |
| if (!virt_addr_valid(to)) { |
| STAT_INC(err_addr); |
| goto out; |
| } |
| |
| if (to & 31) |
| mvOsCacheLineFlushInv(0, to); |
| |
| if ((to+n) & 31) |
| mvOsCacheLineFlushInv(0, to+n); |
| |
| dmac_inv_range((void*)to, (void*)to + n); |
| |
| chain = &xor_chains[CHAIN_FROM_USR]; |
| if (xor_try_chain(chain)) |
| goto out; |
| |
| local_irq_save(flags); |
| |
| while(n) { |
| copy = PAGE_SIZE - (from & ~PAGE_MASK); |
| if (copy > n) |
| copy = n; |
| |
| if (!copy) |
| break; |
| |
| if (copy < XOR_MIN_COPY_CHUNK) { |
| rc = __arch_copy_from_user((void*)to, (void*)from, copy); |
| if (rc) |
| goto out2; |
| } |
| else { |
| |
| sa = __pa_user(from, 0); |
| da = __pa(to); |
| |
| if (unlikely(!sa || !da)) { |
| STAT_INC(err_addr); |
| goto out2; |
| } |
| |
| DPRINTK("%s: desc[%d] %x<-%x %d bytes\n", __FUNCTION__, i, da, sa, copy); |
| |
| dmac_clean_range((void*)from, (void*)from + copy); |
| |
| chain->desc[i].srcAdd0 = sa; |
| chain->desc[i].phyDestAdd = da; |
| chain->desc[i].byteCnt = copy; |
| chain->desc[i].phyNextDescPtr = 0; |
| chain->desc[i].status = BIT31; |
| if (i) |
| chain->desc[i-1].phyNextDescPtr = |
| chain->base + (sizeof(xor_desc_t) * i); |
| |
| i++; |
| } |
| |
| from += copy; |
| to += copy; |
| n -= copy; |
| } |
| |
| if (i) { |
| |
| while(i--) |
| dmac_clean_dcache_line(chain->desc + i); |
| dsb(); |
| |
| xch = xor_get(); |
| if (!xch) |
| goto out2; |
| |
| xor_attach(chain, xch); |
| |
| DPRINTK("%s: ch[%d] chain=%d\n", __FUNCTION__, xch->idx, chain->idx); |
| |
| STAT_INC(from_usr_dma); |
| xor_dma(xch->idx, chain->base); |
| xor_wait(xch->idx); |
| |
| xor_detach(chain, xch); |
| xor_put(xch); |
| } |
| |
| local_irq_restore(flags); |
| |
| return 0; |
| |
| out2: |
| chain->busy=0; |
| local_irq_restore(flags); |
| out: |
| return __arch_copy_from_user((void*)to, (void*)from, n); |
| } |
| EXPORT_SYMBOL(xor_copy_from_user); |
| #else |
| #error "Kernel version >= 2,6,26 does not support XOR copy_from_user" |
| #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) */ |
| #endif /* CONFIG_MV_XOR_COPY_FROM_USER */ |
| |
| |
| #ifdef XOR_BENCH |
| void xor_bench(void *to, void *from, unsigned long n) |
| { |
| unsigned long time; |
| u32 i; |
| u32 t; |
| // |
| // functions under benchmark, use / to skip. |
| // |
| char* fn[] = {"/", "asm_memzero", "/", |
| "/xor_memcpy", "/memcpy", |
| "/xor_memxor", "/xor_block", NULL}; |
| void* va[3]; |
| |
| for (n=128; n<=0x100000; n*=2) { |
| |
| va[0] = kmalloc(n, GFP_KERNEL);; |
| va[1] = kmalloc(n, GFP_KERNEL);; |
| va[2] = kmalloc(n, GFP_KERNEL);; |
| |
| if (!va[1] || !va[0]) { |
| printk("%s: kmalloc failed on %luB\n", __FUNCTION__, n); |
| return; |
| } |
| |
| for (t=0; fn[t]; t++) { |
| if (fn[t][0] == '/') |
| continue; |
| |
| cond_resched(); |
| i = 0; |
| time = jiffies + msecs_to_jiffies(1000); |
| |
| while (jiffies <= time) { |
| switch (t) { |
| case 1: |
| #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) |
| dmac_inv_range((void*)va[0], (void*)va[0] + n); |
| #else |
| dma_cache_maint((void*)va[0], n, DMA_FROM_DEVICE); |
| #endif |
| memzero(va[0], n); |
| break; |
| #ifdef CONFIG_MV_XOR_MEMCOPY |
| case 3: xor_memcpy(va[0], va[1], n); |
| break; |
| case 4: asm_memcpy(va[0], va[1], n); |
| break; |
| #else |
| case 4: memcpy(va[0], va[1], n); |
| break; |
| #endif |
| #ifdef CONFIG_MV_XOR_MEMXOR |
| case 5: xor_memxor(3, n ,va); |
| break; |
| #endif |
| case 6: xor_block(3, n ,va); |
| break; |
| default: |
| printk("%s skipped\n", fn[t]); |
| time = 0; |
| } |
| |
| i++; |
| } |
| |
| printk("%s: %lu8B %8luMBs %s()\n", __FUNCTION__, |
| n, (i * n) >> 20, fn[t]); |
| |
| } |
| |
| kfree(va[2]); |
| kfree(va[1]); |
| kfree(va[0]); |
| } |
| } |
| #endif |
| |