blob: 7a63cc09ea38f2a0244a807bcf116857711b0ed9 [file] [log] [blame]
/*
* (C) Copyright 2012 Quantenna Communications Inc.
*
* See file CREDITS for list of people who contributed to this
* project.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston,
* MA 02111-1307 USA
*/
#ifndef __TOPAZ_HBM_CPUIF_PLATFORM_H
#define __TOPAZ_HBM_CPUIF_PLATFORM_H
#include "mproc_sync.h"
#include "qtn_buffers.h"
#include "qtn_arc_processor.h"
/**
* HBM Requestors
* 0 - LHost
* 1 - MuC
* 2 - emac0 hw wired
* 3 - emac1 hw wired
* 4 - wmac hw wired
* 5 - tqe hw wired
* 6 - AuC
* 7 - DSP
* 8 - PCIE(?)
*
* Note: qdrv_pktlogger_get_hbm_stats must be updated if this list is changed.
*/
#define TOPAZ_HBM_REQUESTOR_NAMES { "lhost", "muc", "emac0", "emac1", "wmac", "tqe", "AuC", "DSP", "PCIe" }
#if defined(__linux__)
#define TOPAZ_HBM_LOCAL_CPU 0
#define topaz_hbm_local_irq_save local_irq_save
#define topaz_hbm_local_irq_restore local_irq_restore
#elif defined(MUC_BUILD)
#define TOPAZ_HBM_LOCAL_CPU 1
#define topaz_hbm_local_irq_save(x) do { (x) = _save_disable(); } while(0)
#define topaz_hbm_local_irq_restore(x) do { _restore_enable((x)); } while(0)
#elif defined(AUC_BUILD)
#define TOPAZ_HBM_LOCAL_CPU 6
#define topaz_hbm_local_irq_save(x) do { (void)(x); } while(0)
#define topaz_hbm_local_irq_restore(x) do { (void)(x); } while(0)
#elif defined(DSP_BUILD)
#define TOPAZ_HBM_LOCAL_CPU 7
#else
#error No TOPAZ_HBM_LOCAL_CPU set
#endif
#define TOPAZ_HBM_NUM_POOLS 4
#ifdef QTN_RC_ENABLE_HDP
#define TOPAZ_HBM_PAYLOAD_HEADROOM 128
#else
#define TOPAZ_HBM_PAYLOAD_HEADROOM 64
#endif
#define TOPAZ_HBM_PAYLOAD_END_GUARD_SIZE 32 /* equal to cacheline size */
#define TOPAZ_HBM_BUF_GUARD_MAGIC 0xDEADBEEF
#define TOPAZ_HBM_BUF_CORRUPTED_MAGIC 0xDEADBEEF
#define TOPAZ_HBM_BUF_PAYLOAD_POISON 0xA5
#define TOPAZ_HBM_BUF_PAYLOAD_POISON32 (TOPAZ_HBM_BUF_PAYLOAD_POISON | \
(TOPAZ_HBM_BUF_PAYLOAD_POISON << 8) | \
(TOPAZ_HBM_BUF_PAYLOAD_POISON << 16) | \
(TOPAZ_HBM_BUF_PAYLOAD_POISON << 24))
#define TOPAZ_HBM_BUF_EXTERNAL_META 1 /* move meta data outside of buffer */
#define TOPAZ_HBM_BUF_WMAC_RX_QUARANTINE 1 /* quarantine wmac rx buffers when deliver important packets */
#if defined(CONFIG_TOPAZ_PCIE_TARGET)
/*
* Checking emac rx pool buffers lead to performance impact in PCIe, so limit
* the check for wmac rx pool only.
*/
#define TOPAZ_HBM_BUF_MAGIC_CHK_ALLPOOL 0
#else
#define TOPAZ_HBM_BUF_MAGIC_CHK_ALLPOOL 1
#endif
#define TOPAZ_HBM_DEBUG_DUMP 0 /* inline debug dump functions */
#define TOPAZ_HBM_DEBUG_STAMPS 0 /* extra trace info in meta data */
#if TOPAZ_HBM_DEBUG_STAMPS
#define TOPAZ_HBM_OWNER_MUC_FREE 0x9
#define TOPAZ_HBM_OWNER_AUC_FREE 0xa
#define TOPAZ_HBM_OWNER_LH_TX_TQE 0xb
#define TOPAZ_HBM_OWNER_LH_RX_TQE 0xc
#define TOPAZ_HBM_OWNER_LH_RX_MBOX 0xd
#define TOPAZ_HBM_OWNER_INIT 0xe
#define TOPAZ_HBM_OWNER_FREE 0xf
#endif
#define TOPAZ_HBM_ERR_NONE 0
#define TOPAZ_HBM_ERR_PTR (-1)
#define TOPAZ_HBM_ERR_MAGIC (-2)
#define TOPAZ_HBM_ERR_TAILGUARD (-3)
/*
* The usage of the HBM buffer headroom and meta data, depending on TOPAZ_HBM_BUF_EXTERNAL_META:
* 1. When it is 1, all the below meta data except magic in head and tail, and pointer to meta are
* in separate memory region, outside of the buffer.
* 2. When it is 0, all the below meta data are in HBM buffer headroom.
* To avoid define different offset values for above 2 cases, we use the same definition. The only
* difference is where the meta data is stored.
*/
enum QTN_HBM_BUF_HEADROOM_OFFSET {
HBM_HR_OFFSET_ENQ_CNT = 1, /* in word */
HBM_HR_OFFSET_FREE_CNT = 2, /* in word */
HBM_HR_OFFSET_OCS_FRM_ID = 3, /* in word */
HBM_HR_OFFSET_FREE_JIFF = 4, /* debugging; jiffies of last free. leaked buffer heuristic */
HBM_HR_OFFSET_OWNER = 5, /* debugging; buffer owner */
HBM_HR_OFFSET_SIZE = 6, /* debugging; buffer size */
HBM_HR_OFFSET_STATE = 7, /* state about the buffer */
HBM_HR_OFFSET_META_PTR = 8, /* pointer and back pointer bwtween buffer and meta, bus addr */
HBM_HR_OFFSET_MAGIC = 9, /* the magic, keep it biggest thus first in headroom */
HBM_HR_OFFSET_MAX = HBM_HR_OFFSET_MAGIC,
};
#define QTN_HBM_SANITY_BAD_HR_MAGIC BIT(0)
#define QTN_HBM_SANITY_BAD_TAIL_GUARD BIT(1)
#define QTN_HBM_SANITY_BAD_ALREADY BIT(31)
#define TOPAZ_HBM_BUF_DUMP_MAX 0xFFFF
#define TOPAZ_HBM_BUF_DUMP_DFT 512U
#define TOPAZ_HBM_BUF_DUMP_TAIL_DFT 2048U
#if TOPAZ_HBM_DEBUG_DUMP
#if defined(DSP_BUILD)
#elif defined(AUC_BUILD)
#define CPU_PRINT auc_os_printf
#define CPU_INV_DCACHE_RANGE(_v, _range)
#define CPU_HZ AUC_CPU_TIMER_HZ
#elif defined(MUC_BUILD)
#define CPU_PRINT uc_printk
#define CPU_INV_DCACHE_RANGE invalidate_dcache_range_safe
#define CPU_HZ HZ
#else
#ifdef __KERNEL__
#include <qtn/dmautil.h>
#define CPU_PRINT printk
#define CPU_INV_DCACHE_RANGE inv_dcache_sizerange_safe
#define CPU_HZ HZ
#endif
#endif
#endif // TOPAZ_HBM_DEBUG_DUMP
RUBY_INLINE uint32_t topaz_hbm_buf_offset_from_start_bus(void *buf_bus, uint8_t pool, uint8_t is_aligned)
{
if (is_aligned) {
return ((uint32_t)buf_bus) & (TOPAZ_HBM_BUF_ALIGN - 1);
}
if (pool == TOPAZ_HBM_BUF_WMAC_RX_POOL) {
return (((uint32_t)buf_bus) - TOPAZ_HBM_BUF_WMAC_RX_BASE) % TOPAZ_HBM_BUF_WMAC_RX_SIZE;
} else if (pool == TOPAZ_HBM_BUF_EMAC_RX_POOL) {
return (((uint32_t)buf_bus) - TOPAZ_HBM_BUF_EMAC_RX_BASE) % TOPAZ_HBM_BUF_EMAC_RX_SIZE;
} else {
return 0;
}
}
RUBY_INLINE int topaz_hbm_buf_identify_buf_bus(const void *buf_bus, uint32_t *sizep, uint32_t *idxp)
{
if (__in_mem_range((uint32_t)buf_bus, TOPAZ_HBM_BUF_EMAC_RX_BASE, TOPAZ_HBM_BUF_EMAC_RX_TOTAL)) {
*sizep = TOPAZ_HBM_BUF_EMAC_RX_SIZE;
*idxp = (((uint32_t)buf_bus) - TOPAZ_HBM_BUF_EMAC_RX_BASE) / TOPAZ_HBM_BUF_EMAC_RX_SIZE;
return TOPAZ_HBM_BUF_EMAC_RX_POOL;
} else if (__in_mem_range((uint32_t)buf_bus, TOPAZ_HBM_BUF_WMAC_RX_BASE, TOPAZ_HBM_BUF_WMAC_RX_TOTAL)) {
*sizep = TOPAZ_HBM_BUF_WMAC_RX_SIZE;
*idxp = (((uint32_t)buf_bus) - TOPAZ_HBM_BUF_WMAC_RX_BASE) / TOPAZ_HBM_BUF_WMAC_RX_SIZE;
return TOPAZ_HBM_BUF_WMAC_RX_POOL;
} else {
return -1;
}
}
RUBY_INLINE int topaz_hbm_buf_identify_buf_virt(const void *buf_virt, uint32_t *sizep, uint32_t *idxp)
{
if (__in_mem_range((uint32_t)buf_virt, TOPAZ_HBM_BUF_EMAC_RX_BASE_VIRT, TOPAZ_HBM_BUF_EMAC_RX_TOTAL)) {
*sizep = TOPAZ_HBM_BUF_EMAC_RX_SIZE;
*idxp = (((uint32_t)buf_virt) - TOPAZ_HBM_BUF_EMAC_RX_BASE_VIRT) / TOPAZ_HBM_BUF_EMAC_RX_SIZE;
return TOPAZ_HBM_BUF_EMAC_RX_POOL;
} else if (__in_mem_range((uint32_t)buf_virt, TOPAZ_HBM_BUF_WMAC_RX_BASE_VIRT, TOPAZ_HBM_BUF_WMAC_RX_TOTAL)) {
*sizep = TOPAZ_HBM_BUF_WMAC_RX_SIZE;
*idxp = (((uint32_t)buf_virt) - TOPAZ_HBM_BUF_WMAC_RX_BASE_VIRT) / TOPAZ_HBM_BUF_WMAC_RX_SIZE;
return TOPAZ_HBM_BUF_WMAC_RX_POOL;
} else {
return -1;
}
}
RUBY_INLINE int topaz_hbm_buf_ptr_valid(const void *buf_virt)
{
uint32_t offset;
if (__in_mem_range((uint32_t)buf_virt, TOPAZ_HBM_BUF_EMAC_RX_BASE_VIRT, TOPAZ_HBM_BUF_EMAC_RX_TOTAL)) {
offset = (((uint32_t)buf_virt) - TOPAZ_HBM_BUF_EMAC_RX_BASE_VIRT) % TOPAZ_HBM_BUF_EMAC_RX_SIZE;
} else if (__in_mem_range((uint32_t)buf_virt, TOPAZ_HBM_BUF_WMAC_RX_BASE_VIRT, TOPAZ_HBM_BUF_WMAC_RX_TOTAL)) {
offset = (((uint32_t)buf_virt) - TOPAZ_HBM_BUF_WMAC_RX_BASE_VIRT) % TOPAZ_HBM_BUF_WMAC_RX_SIZE;
} else {
return 0;
}
return (offset == TOPAZ_HBM_PAYLOAD_HEADROOM);
}
#if TOPAZ_HBM_BUF_EXTERNAL_META
RUBY_INLINE void* topaz_hbm_buf_get_meta(const void *buf_virt)
{
uint32_t idx;
if (__in_mem_range((uint32_t)buf_virt, TOPAZ_HBM_BUF_EMAC_RX_BASE_VIRT, TOPAZ_HBM_BUF_EMAC_RX_TOTAL)) {
idx = (((uint32_t)buf_virt) - TOPAZ_HBM_BUF_EMAC_RX_BASE_VIRT) / TOPAZ_HBM_BUF_EMAC_RX_SIZE;
return (void*)(TOPAZ_HBM_BUF_META_EMAC_RX_BASE_VIRT + TOPAZ_HBM_BUF_META_SIZE +
idx * TOPAZ_HBM_BUF_META_SIZE);
} else if (__in_mem_range((uint32_t)buf_virt, TOPAZ_HBM_BUF_WMAC_RX_BASE_VIRT, TOPAZ_HBM_BUF_WMAC_RX_TOTAL)) {
idx = (((uint32_t)buf_virt) - TOPAZ_HBM_BUF_WMAC_RX_BASE_VIRT) / TOPAZ_HBM_BUF_WMAC_RX_SIZE;
return (void*)(TOPAZ_HBM_BUF_META_WMAC_RX_BASE_VIRT + TOPAZ_HBM_BUF_META_SIZE +
idx * TOPAZ_HBM_BUF_META_SIZE);
} else {
return NULL;
}
}
/*
* A fast way to get the meta address.
* However this assume the meta address in buffer headroom is not corrupted as long as the magic in
* buffer headroom is not corrupted. So this is not 100% correct, but it can be used to speed up for
* some non-real-world test.
*/
RUBY_INLINE void* topaz_hbm_buf_get_meta_fast(const void *buf_virt)
{
uint32_t *magicp = (uint32_t*)buf_virt - HBM_HR_OFFSET_MAGIC;
/* assume ptr is valid when magic is not corrupted */
if (likely(arc_read_uncached_32(magicp) == TOPAZ_HBM_BUF_GUARD_MAGIC)) {
uint32_t *meta_ptr_p = (uint32_t*)buf_virt - HBM_HR_OFFSET_META_PTR;
return (void*)bus_to_virt(arc_read_uncached_32(meta_ptr_p));
}
return topaz_hbm_buf_get_meta(buf_virt);
}
#else
#define topaz_hbm_buf_get_meta_fast(_buf_virt) (_buf_virt)
#define topaz_hbm_buf_get_meta(_buf_virt) (_buf_virt)
#endif
RUBY_INLINE uint32_t topaz_hbm_buf_offset_from_start_virt(void *buf_virt, uint8_t pool, uint8_t is_aligned)
{
return topaz_hbm_buf_offset_from_start_bus((void *)virt_to_bus(buf_virt), pool, is_aligned);
}
RUBY_INLINE void *topaz_hbm_payload_store_align_bus(void *buf_bus, uint8_t pool, uint8_t is_aligned)
{
return ((uint8_t *)buf_bus) - topaz_hbm_buf_offset_from_start_bus(buf_bus, pool, is_aligned)
+ TOPAZ_HBM_PAYLOAD_HEADROOM;
}
RUBY_INLINE void *topaz_hbm_payload_store_align_virt(void *buf_virt, uint8_t pool, uint8_t is_aligned)
{
return ((uint8_t *)buf_virt) - topaz_hbm_buf_offset_from_start_virt(buf_virt, pool, is_aligned)
+ TOPAZ_HBM_PAYLOAD_HEADROOM;
}
RUBY_INLINE unsigned long topaz_hbm_payload_store_align_from_index(int8_t pool, uint16_t index)
{
if (pool == TOPAZ_HBM_BUF_EMAC_RX_POOL && index < TOPAZ_HBM_BUF_EMAC_RX_COUNT) {
return RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_EMAC_RX_BASE +
(index * TOPAZ_HBM_BUF_EMAC_RX_SIZE) + TOPAZ_HBM_PAYLOAD_HEADROOM;
} else if (pool == TOPAZ_HBM_BUF_WMAC_RX_POOL && index < TOPAZ_HBM_BUF_WMAC_RX_COUNT) {
return RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_WMAC_RX_BASE +
(index * TOPAZ_HBM_BUF_WMAC_RX_SIZE) + TOPAZ_HBM_PAYLOAD_HEADROOM;
}
return 0;
}
RUBY_INLINE long topaz_hbm_payload_buff_ptr_offset_bus(void *buf_bus, uint8_t pool, void *align_bus)
{
unsigned long buf_align = (unsigned long)topaz_hbm_payload_store_align_bus(
align_bus ? align_bus : buf_bus, pool, !!align_bus);
return buf_align - (unsigned long)buf_bus;
}
RUBY_INLINE long topaz_hbm_payload_buff_ptr_offset_virt(void *buf_virt, uint8_t pool, void *align_virt)
{
return topaz_hbm_payload_buff_ptr_offset_bus((void *)virt_to_bus(buf_virt), pool,
align_virt ? (void *)virt_to_bus(align_virt) : NULL);
}
#if defined(TOPAZ_PLATFORM)
RUBY_INLINE int __topaz_hbm_is_done(void)
{
return qtn_mproc_sync_mem_read(TOPAZ_HBM_POOL_REQ(TOPAZ_HBM_LOCAL_CPU)) & TOPAZ_HBM_DONE;
}
RUBY_INLINE void __topaz_hbm_release_buf(void *buf, uint8_t pool)
{
/* assumes previous operations are complete */
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_POOL_DATA(TOPAZ_HBM_LOCAL_CPU),
(unsigned long) buf);
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_POOL_REQ(TOPAZ_HBM_LOCAL_CPU),
TOPAZ_HBM_POOL_NUM(pool) | TOPAZ_HBM_RELEASE_BUF);
}
RUBY_INLINE void __topaz_hbm_request_start(uint8_t pool)
{
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_POOL_REQ(TOPAZ_HBM_LOCAL_CPU),
TOPAZ_HBM_POOL_NUM(pool) | TOPAZ_HBM_REQUEST_BUF);
}
RUBY_INLINE void *__topaz_hbm_rd_buf(uint8_t pool)
{
/* must be preceded by __topaz_hbm_rd_req, then polling on __topaz_hbm_is_done */
return (void *) qtn_mproc_sync_mem_read(TOPAZ_HBM_POOL_DATA(TOPAZ_HBM_LOCAL_CPU));
}
RUBY_INLINE void __topaz_hbm_wait(void)
{
unsigned int timeouts = 0;
unsigned int timeout_reached = 0;
while (1) {
int i;
if (__topaz_hbm_is_done()) {
return;
}
/* busy wait until buffer is available */
for (i = 0; i < 10; i++) {
#if defined(AUC_BUILD) && defined(_ARC)
/*
* This is a workaround for MetaWare C Compiler v7.4.0
* bug in Zero-Delay Loop code generation for ARC 600 family cores.
* Without it the LP_START register will be written to, two
* instruction before the end address of the loop, but at least three
* instructions are required according to the ARC ISA Programmer's
* Reference.
*/
_nop();
_nop();
_nop();
#else
qtn_pipeline_drain();
#endif
}
if (unlikely(timeout_reached == 0 && timeouts++ == 1000)) {
timeout_reached = 1;
qtn_mproc_sync_log("__topaz_hbm_wait timeout");
}
}
if (unlikely(timeout_reached)) {
qtn_mproc_sync_log("__topaz_hbm_wait succeeded");
}
}
RUBY_INLINE int __topaz_hbm_put_buf_nowait(void *buf, uint8_t pool)
{
if (__topaz_hbm_is_done()) {
__topaz_hbm_release_buf(buf, pool);
return 0;
}
return -EBUSY;
}
RUBY_INLINE void __topaz_hbm_put_buf(void *buf, uint8_t pool)
{
__topaz_hbm_wait();
__topaz_hbm_release_buf(buf, pool);
}
RUBY_WEAK(topaz_hbm_put_buf) void topaz_hbm_put_buf(void *buf, uint8_t pool)
{
unsigned long flags;
topaz_hbm_local_irq_save(flags);
__topaz_hbm_put_buf(buf, pool);
topaz_hbm_local_irq_restore(flags);
}
RUBY_INLINE void *__topaz_hbm_get_buf(uint8_t pool)
{
__topaz_hbm_wait();
__topaz_hbm_request_start(pool);
__topaz_hbm_wait();
return __topaz_hbm_rd_buf(pool);
}
RUBY_WEAK(topaz_hbm_get_buf) void *topaz_hbm_get_buf(uint8_t pool)
{
unsigned long flags;
void *buf;
topaz_hbm_local_irq_save(flags);
buf = __topaz_hbm_get_buf(pool);
topaz_hbm_local_irq_restore(flags);
return buf;
}
RUBY_INLINE void topaz_hbm_init(void *pool_list_bus, uint16_t payload_count_s, uint8_t pool, int full)
{
unsigned long csr;
const uint16_t payload_count = BIT(payload_count_s);
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_BASE_REG(pool), (unsigned long) pool_list_bus);
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_LIMIT_REG(pool), payload_count);
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_WR_PTR(pool), full ? payload_count : 0);
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_RD_PTR(pool), 0);
csr = qtn_mproc_sync_mem_read(TOPAZ_HBM_CSR_REG);
qtn_mproc_sync_mem_write_wmb(TOPAZ_HBM_CSR_REG, csr | TOPAZ_HBM_CSR_Q_EN(pool));
}
#endif /* defined(TOPAZ_PLATFORM) */
#if defined(TOPAZ_PLATFORM) || defined(RUBY_SOFTWARE_HBM)
RUBY_INLINE uint32_t topaz_hbm_pool_buf_whole_size(int8_t pool)
{
if (pool == TOPAZ_HBM_BUF_EMAC_RX_POOL) {
return TOPAZ_HBM_BUF_EMAC_RX_SIZE;
} else if (pool == TOPAZ_HBM_BUF_WMAC_RX_POOL) {
return TOPAZ_HBM_BUF_WMAC_RX_SIZE;
} else {
return 0;
}
}
RUBY_INLINE uint32_t topaz_hbm_pool_buf_max_size(int8_t pool)
{
uint32_t size;
size = topaz_hbm_pool_buf_whole_size(pool);
if (!size)
return 0;
return size -
TOPAZ_HBM_PAYLOAD_HEADROOM -
TOPAZ_HBM_PAYLOAD_END_GUARD_SIZE;
}
RUBY_INLINE int8_t topaz_hbm_pool_valid(int8_t pool)
{
return pool >= 0 && pool < TOPAZ_HBM_NUM_POOLS;
}
RUBY_INLINE int8_t topaz_hbm_payload_get_pool_bus(const void *buf_bus)
{
const unsigned long v = (const unsigned long) buf_bus;
if (__in_mem_range(v, TOPAZ_HBM_BUF_EMAC_RX_BASE, TOPAZ_HBM_BUF_EMAC_RX_TOTAL)) {
return TOPAZ_HBM_BUF_EMAC_RX_POOL;
} else if (__in_mem_range(v, TOPAZ_HBM_BUF_WMAC_RX_BASE, TOPAZ_HBM_BUF_WMAC_RX_TOTAL)) {
return TOPAZ_HBM_BUF_WMAC_RX_POOL;
} else {
return -1;
}
}
RUBY_INLINE int8_t topaz_hbm_payload_get_free_pool_bus(const void *buf_bus)
{
int8_t orig_pool = topaz_hbm_payload_get_pool_bus(buf_bus);
if (orig_pool == TOPAZ_HBM_BUF_EMAC_RX_POOL || orig_pool == TOPAZ_HBM_BUF_WMAC_RX_POOL) {
return TOPAZ_HBM_EMAC_TX_DONE_POOL;
}
return -1;
}
RUBY_INLINE void topaz_hbm_put_payload_aligned_bus(void *buf_bus, int8_t pool)
{
if (likely(topaz_hbm_pool_valid(pool))) {
topaz_hbm_put_buf(topaz_hbm_payload_store_align_bus(buf_bus, pool, 1), pool);
}
}
RUBY_INLINE void topaz_hbm_put_payload_realign_bus(void *buf_bus, int8_t pool)
{
if (likely(topaz_hbm_pool_valid(pool))) {
topaz_hbm_put_buf(topaz_hbm_payload_store_align_bus(buf_bus, pool, 0), pool);
}
}
RUBY_INLINE void topaz_hbm_put_payload_aligned_virt(void *buff_virt, int8_t pool)
{
topaz_hbm_put_payload_aligned_bus((void *) virt_to_bus(buff_virt), pool);
}
RUBY_INLINE void topaz_hbm_put_payload_realign_virt(void *buff_virt, int8_t pool)
{
topaz_hbm_put_payload_realign_bus((void *) virt_to_bus(buff_virt), pool);
}
#ifdef __KERNEL__
#define topaz_hbm_get_payload_bus(pool) __topaz_hbm_get_payload_bus((pool), __FILE__, __LINE__, __FUNCTION__)
RUBY_INLINE void *__topaz_hbm_get_payload_bus(int8_t pool, const char *file, const int line, const char *func)
{
if (likely(topaz_hbm_pool_valid(pool))) {
return topaz_hbm_get_buf(pool);
}
if (printk_ratelimit()) {
printk("%s:%u%s null buffer from pool %hhd\n",
file, line, func, pool);
}
return NULL;
}
#define topaz_hbm_get_payload_virt(pool) __topaz_hbm_get_payload_virt((pool), __FILE__, __LINE__, __FUNCTION__)
RUBY_INLINE void *__topaz_hbm_get_payload_virt(int8_t pool, const char *file, const int line, const char *func)
{
void *buf_bus = topaz_hbm_get_payload_bus(pool);
if (unlikely(!buf_bus)) {
if (printk_ratelimit()) {
printk("%s:%u%s null buffer from pool %hhd\n",
file, line, func, pool);
}
return NULL;
}
return bus_to_virt((unsigned long) buf_bus);
}
#else
RUBY_INLINE void *topaz_hbm_get_payload_bus(int8_t pool)
{
if (likely(topaz_hbm_pool_valid(pool))) {
return topaz_hbm_get_buf(pool);
}
return NULL;
}
RUBY_INLINE void *topaz_hbm_get_payload_virt(int8_t pool)
{
void *buf_bus = topaz_hbm_get_payload_bus(pool);
if (unlikely(!buf_bus)) {
return NULL;
}
return bus_to_virt((unsigned long) buf_bus);
}
#endif
RUBY_INLINE int hbm_buf_check_wmac_rx_buf_overrun(void *v, int fix)
{
uint32_t *guardp;
/* only check last 4 bytes guard */
guardp =(uint32_t*)((uint32_t)v + TOPAZ_HBM_BUF_WMAC_RX_SIZE - TOPAZ_HBM_PAYLOAD_HEADROOM - 4);
if (likely(arc_read_uncached_32(guardp) == TOPAZ_HBM_BUF_GUARD_MAGIC)) {
return TOPAZ_HBM_ERR_NONE;
}
/*
* It is best if we do the buffer pointer check first, but as we only do the overrun check after wmac rx,
* if it is bad, the memory is already corrupted.
*/
if (fix) {
arc_write_uncached_32(guardp, TOPAZ_HBM_BUF_GUARD_MAGIC);
}
return TOPAZ_HBM_ERR_TAILGUARD;
}
RUBY_INLINE int hbm_buf_check_buf_magic(void *v)
{
uint32_t *magicp = (uint32_t*)v - HBM_HR_OFFSET_MAGIC;
if (likely(arc_read_uncached_32(magicp) == TOPAZ_HBM_BUF_GUARD_MAGIC)) {
return TOPAZ_HBM_ERR_NONE;
}
return TOPAZ_HBM_ERR_MAGIC;
}
RUBY_INLINE void hbm_buf_fix_buf_magic(void *v)
{
uint32_t *magicp = (uint32_t*)v - HBM_HR_OFFSET_MAGIC;
arc_write_uncached_32(magicp, TOPAZ_HBM_BUF_GUARD_MAGIC);
}
#if TOPAZ_HBM_DEBUG_DUMP
/* assume v is 4 bytes aligned */
RUBY_INLINE void topaz_buf_dump_range(const void *v, int len)
{
#if defined(DSP_BUILD)
#elif defined(AUC_BUILD)
int i;
const uint32_t *d32;
int dump_loop;
d32 = v;
dump_loop = ((len + 3) >> 2) >> 3;
for (i = 0; i < dump_loop; i++) {
CPU_PRINT("0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x",
d32[0], d32[1], d32[2], d32[3], d32[4], d32[5], d32[6], d32[7]);
d32 += 8;
}
#else
int i;
const uint8_t *d;
d = v;
for (i = 0; i < len; ) {
if (!(i % 32))
CPU_PRINT("%08x ", (i - i % 32));
++i;
CPU_PRINT("%02x%s", *d++, (i % 32) == 0 ? "\n" : " ");
}
CPU_PRINT("\n");
#endif
}
RUBY_INLINE void topaz_hbm_buf_show(void *v, const uint32_t len, const uint32_t tail_len)
{
#if defined(DSP_BUILD)
#else
const uint32_t *_p = v;
const uint32_t *_m = topaz_hbm_buf_get_meta(_p);
const uint32_t *enqueuep = _m - HBM_HR_OFFSET_ENQ_CNT;
const uint32_t *freep = _m - HBM_HR_OFFSET_FREE_CNT;
const uint32_t *jiffp = _m - HBM_HR_OFFSET_FREE_JIFF;
const uint32_t *ownerp = _m - HBM_HR_OFFSET_OWNER;
const uint32_t *sizep = _m - HBM_HR_OFFSET_SIZE;
const uint32_t *magicp = _p - HBM_HR_OFFSET_MAGIC;
const uint32_t ec = arc_read_uncached_32(enqueuep);
const uint32_t fc = arc_read_uncached_32(freep);
const uint32_t jc = arc_read_uncached_32(jiffp);
const uint32_t oc = arc_read_uncached_32(ownerp);
const uint32_t sz = arc_read_uncached_32(sizep);
const uint32_t magic = arc_read_uncached_32(magicp);
uint32_t *guardp;
uint32_t guard;
int dump_bytes;
uint32_t whole_size;
uint32_t payload_size;
int pool;
uint32_t idx;
pool = topaz_hbm_buf_identify_buf_virt(v, &whole_size, &idx);
if (pool < 0) {
return;
}
payload_size = whole_size - TOPAZ_HBM_PAYLOAD_HEADROOM;
CPU_INV_DCACHE_RANGE((void*)v, payload_size);
dump_bytes = (len == TOPAZ_HBM_BUF_DUMP_MAX) ? payload_size : len;
/* only check last 4 bytes guard */
guardp =(uint32_t*)((uint32_t)_p + payload_size - 4);
guard = arc_read_uncached_32(guardp);
CPU_PRINT("buf start 0x%x pool %d idx %u size %u dump %u\n",
(unsigned int)v, pool, idx, whole_size, dump_bytes);
#ifdef __KERNEL__
CPU_PRINT("%p ec %u fp %u own %08x size %u j %u (%u s ago)\n",
v, ec, fc, oc, sz, jc, (((uint32_t) jiffies) - jc) / CPU_HZ);
#else
/* free jiffies is only set by Lhost, so no way to do jiffies diff */
CPU_PRINT("%p ec %u fp %u own %08x size %u j %u (local j %u)\n",
v, ec, fc, oc, sz, jc, ((uint32_t) jiffies));
#endif
if (magic != TOPAZ_HBM_BUF_GUARD_MAGIC) {
CPU_PRINT("magic %x corrupted\n", magic);
}
if (guard != TOPAZ_HBM_BUF_GUARD_MAGIC) {
CPU_PRINT("guard %x corrupted\n", guard);
}
topaz_buf_dump_range(v, dump_bytes);
if (tail_len) {
uint32_t tail;
tail = (uint32_t)v;
tail += payload_size;
tail -= tail_len;
CPU_PRINT("buf tail 0x%x\n", tail);
topaz_buf_dump_range((void*)tail, tail_len);
}
#endif
}
/*
* Full sanity check suitable for all buffers.
* Debug build use only, not suitable for release because of performance impact.
*/
RUBY_INLINE int hbm_buf_check_sanity(void *v)
{
const uint32_t *magicp = (uint32_t*)v - HBM_HR_OFFSET_MAGIC;
uint32_t *_m;
uint32_t *statep;
uint32_t state;
uint32_t magic;
uint32_t payload_size;
uint32_t *guardp;
uint32_t guard;
uint32_t size = 0;
uint32_t idx = 0;
uint32_t bad = 0;
int pool;
magic = arc_read_uncached_32(magicp);
if (unlikely(magic != TOPAZ_HBM_BUF_GUARD_MAGIC)) {
bad |= QTN_HBM_SANITY_BAD_HR_MAGIC;
}
pool = topaz_hbm_buf_identify_buf_virt(v, &size, &idx);
payload_size = size - TOPAZ_HBM_PAYLOAD_HEADROOM;
/* only check last 4 bytes guard */
guardp =(uint32_t*)((uint32_t)v + payload_size - 4);
guard = arc_read_uncached_32(guardp);
if (unlikely(guard != TOPAZ_HBM_BUF_GUARD_MAGIC)) {
bad |= QTN_HBM_SANITY_BAD_TAIL_GUARD;
}
if (likely(!bad))
return 0;
/* avoid multiple alert */
_m = topaz_hbm_buf_get_meta(v);
statep = (uint32_t*)_m - HBM_HR_OFFSET_STATE;
state = arc_read_uncached_32(statep);
if ((bad & (~state)) == 0) {
return (bad | QTN_HBM_SANITY_BAD_ALREADY);
}
/* new corruption */
arc_write_uncached_32(statep, bad);
CPU_PRINT("ERROR: hbm buffer %x corrupted, pool %d, idx %u\n",
(unsigned int)v, pool, idx);
topaz_hbm_buf_show(v, TOPAZ_HBM_BUF_DUMP_DFT, 0);
/* new corruption of tail guard */
if ((bad & QTN_HBM_SANITY_BAD_TAIL_GUARD) && !(state & QTN_HBM_SANITY_BAD_TAIL_GUARD)) {
/* find the corruption extent */
int i;
int j = 0;
int lines = (size * 4) / 16;
uint32_t pos = 0;
for (i = 0; i < lines ; i++) {
for (j = 0; j < 4; j++) {
pos = (uint32_t)v + i * 16 + j * 4;
if (*(uint32_t*)pos != (uint32_t)TOPAZ_HBM_BUF_PAYLOAD_POISON32)
break;
}
if (j == 4)
break;
}
CPU_PRINT("guess tail corruption length %d %x\n", (i * 16) + (j * 4), pos);
}
return bad;
}
#endif // TOPAZ_HBM_DEBUG_DUMP
#if TOPAZ_HBM_DEBUG_STAMPS
RUBY_INLINE void topaz_hbm_debug_stamp(void *buf, uint8_t port, uint32_t size)
{
uint32_t *p = buf;
uint32_t *_m = topaz_hbm_buf_get_meta(p);
uint32_t *ownerp = _m - HBM_HR_OFFSET_OWNER;
uint32_t *sizep = _m - HBM_HR_OFFSET_SIZE;
arc_write_uncached_32(ownerp, (arc_read_uncached_32(ownerp) << 4) | (port & 0xF));
if (size) {
arc_write_uncached_32(sizep, size);
}
}
#else
#define topaz_hbm_debug_stamp(_buf, _port, _size)
#endif /* TOPAZ_HBM_DEBUG_STAMPS */
#endif /* defined(TOPAZ_PLATFORM) || defined(RUBY_SOFTWARE_HBM) */
#endif /* __TOPAZ_HBM_CPUIF_PLATFORM_H */