blob: e027425c4affec064f30eb5e7f2ddd754e96d167 [file] [log] [blame]
/*
* (C) Copyright 2012 Quantenna Communications Inc.
*
* See file CREDITS for list of people who contributed to this
* project.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston,
* MA 02111-1307 USA
*/
/*
* Quantenna HBM skb payload pool
*/
#include <linux/kernel.h>
#include <linux/cache.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/skbuff.h>
#include <qtn/dmautil.h>
#include <asm/io.h>
#include <common/queue.h>
#ifndef TOPAZ_PLATFORM
#include <qtn/ruby_sbm.h>
#endif
#include <qtn/topaz_hbm.h>
#include <qtn/dmautil.h>
#include <net80211/if_ethersubr.h>
#define isspace(c) ((((c) == ' ') || (((unsigned int)((c) - 9)) <= (13 - 9))))
#define TOPAZ_HBM_PROC_FILENAME "topaz_hbm"
#define TOPAZ_HBM_IF_PROC_NAME "topaz_hbm_if"
#define HBM_BUF_DEPLETION_TH (20)
#define HBM_BUF_POLL_S_INTRVAL (10 * HZ)
#define HBM_BUF_POLL_L_INTRVAL (60 * HZ)
#define HBM_BUF_MINIMUM_AVAIL_NUM (3)
#define HBM_BUF_MINIMUM_REL_NUM (10 * HBM_BUF_POLL_S_INTRVAL)
typedef enum hbm_if_usr_cmd {
HBM_IF_CMD_DUMPCTL = 0,
HBM_IF_CMD_STATS = 1,
HBM_IF_MAX_CMD,
} hbm_if_usr_cmd;
#define HBM_IF_KEY_DUMPCTL "dumpctl"
#define HBM_IF_KEY_STATS "stats"
static char* str_cmd[HBM_IF_MAX_CMD] = {
HBM_IF_KEY_DUMPCTL,
HBM_IF_KEY_STATS,
};
typedef enum hbm_stats {
HBM_CNT_INVALID_POOL = 0,
HBM_CNT_MISALIGNED = 1,
HBM_CNT_MAGIC_CORRUPTED = 2,
HBM_CNT_QUARANTINE_CORRUPTED = 3,
HBM_CNT_QUARANTINE_ALLOC_FAIL = 4,
HBM_CNT_QUARANTINE_OK = 5,
HBM_CNT_NUM,
} hbm_stats;
struct hbm_pool_cnt {
uint32_t prev_release_cnt;
uint32_t pool_depleted_cnt;
};
struct topaz_hbm_mnt {
uint32_t prev_unflow_cnt;
uint32_t unflow_flag;
struct hbm_pool_cnt wmac_pl;
struct hbm_pool_cnt emac_pl;
};
static DEFINE_TIMER(hbm_timer, NULL, 0, 0);
static uint32_t topaz_hbm_stats[HBM_CNT_NUM] = {0};
static const char *topaz_hbm_stats_names[HBM_CNT_NUM] = {
"Invalid pool",
"Misaligned pointer",
"Magic corrupted",
"Quarantine corrupted buffer",
"Quarantine allocation fail",
"Quarantine ok",
};
#define HBM_STATS(_idx, _num) (topaz_hbm_stats[(_idx)] += (_num))
#ifdef TOPAZ_EMAC_NULL_BUF_WR
#define HBM_UFLOW_RECOVER_TH 32
void __attribute__((section(".sram.data")))(*topaz_emac_null_buf_del_cb)(void) = NULL;
EXPORT_SYMBOL(topaz_emac_null_buf_del_cb);
#endif
static const char *topaz_hbm_requestor_names[TOPAZ_HBM_MASTER_COUNT] = TOPAZ_HBM_REQUESTOR_NAMES;
unsigned int topaz_hbm_pool_available(int8_t pool)
{
uint32_t wr_ptr;
uint32_t rd_ptr;
static const unsigned int TOPAZ_HBM_MAX_POOL_COUNT = (1 << 16);
if (!topaz_hbm_pool_valid(pool)) {
printk(KERN_ERR"%s: Invalid pool %d\n", __func__, pool);
return TOPAZ_HBM_MAX_POOL_COUNT;
}
wr_ptr = readl(TOPAZ_HBM_WR_PTR(pool));
rd_ptr = readl(TOPAZ_HBM_RD_PTR(pool));
if (wr_ptr >= rd_ptr)
return (wr_ptr - rd_ptr);
else
return (TOPAZ_HBM_MAX_POOL_COUNT - rd_ptr + wr_ptr);
}
EXPORT_SYMBOL(topaz_hbm_pool_available);
#define HBM_DUMP_SORT_ORDER_INV_BASE 100
typedef enum hbm_dump_sort_type {
HBM_DUMP_SORT_ADDR = 0, /* lower addr first */
HBM_DUMP_SORT_JIFF, /* newest freed first */
HBM_DUMP_SORT_BAD_MAGIC, /* bad magic first */
HBM_DUMP_SORT_ADDR_MAX = HBM_DUMP_SORT_ORDER_INV_BASE - 1,
} hbm_dump_sort_type;
static hbm_dump_sort_type topaz_hbm_dump_sort_type = HBM_DUMP_SORT_ADDR;
static int topaz_hbm_dump_sort_range_min = 0; /* meaning dependent on sort type */
static int topaz_hbm_dump_sort_range_max = 0xFFFFFFFF; /* meaning dependent on sort type */
static int topaz_hbm_dump_num = 5; /* max dump number */
static int topaz_hbm_dump_len = 128; /* bytes dump at head */
static int topaz_hbm_dump_taillen = 32; /* bytes dump at tail */
static int topaz_hbm_dumped_num = 0; /* currently dumpped number */
#define TOPAZ_HBM_POOL_SIZE_MAX (TOPAZ_HBM_BUF_EMAC_RX_COUNT + 1)
uint32_t* topaz_hbm_dump_bufs_sorted[TOPAZ_HBM_POOL_SIZE_MAX] = {0};
static int topaz_hbm_stat_rd(char *page, char **start, off_t offset,
int count, int *eof, void *data)
{
char *p = page;
uint32_t wr_ptr;
uint32_t rd_ptr;
#ifdef TOPAZ_PLATFORM
int req_rel_diff = 0;
int req_rel_perpool_diff[TOPAZ_HBM_POOL_COUNT];
int master, pool;
/* offsets for initial pool loading */
req_rel_perpool_diff[TOPAZ_HBM_BUF_EMAC_RX_POOL] = TOPAZ_HBM_BUF_EMAC_RX_COUNT;
req_rel_perpool_diff[TOPAZ_HBM_BUF_WMAC_RX_POOL] = TOPAZ_HBM_BUF_WMAC_RX_COUNT;
req_rel_perpool_diff[TOPAZ_HBM_AUC_FEEDBACK_POOL] = 0;
req_rel_perpool_diff[TOPAZ_HBM_EMAC_TX_DONE_POOL] = 0;
for (pool = 0; pool < TOPAZ_HBM_POOL_COUNT; ++pool) {
for (master = 0; master < TOPAZ_HBM_MASTER_COUNT; ++master) {
uint32_t req = readl(TOPAZ_HBM_POOL_REQUEST_CNT(master, pool));
uint32_t rel = readl(TOPAZ_HBM_POOL_RELEASE_CNT(master, pool));
req_rel_perpool_diff[pool] += req;
req_rel_perpool_diff[pool] -= rel;
p += sprintf(p, "master %5s pool %d req %u rel %u\n",
topaz_hbm_requestor_names[master], pool, req, rel);
}
}
for (pool = 0; pool < TOPAZ_HBM_POOL_COUNT; ++pool) {
req_rel_diff += req_rel_perpool_diff[pool];
wr_ptr = readl(TOPAZ_HBM_WR_PTR(pool));
rd_ptr = readl(TOPAZ_HBM_RD_PTR(pool));
p += sprintf(p, "pool %u rd_ptr %u wr_ptr %u\n", pool, rd_ptr, wr_ptr);
}
uint32_t overflow = readl(TOPAZ_HBM_OVERFLOW_CNT);
uint32_t underflow = readl(TOPAZ_HBM_UNDERFLOW_CNT);
int allocated = req_rel_diff - underflow;
p += sprintf(p, "underflow %u overflow %u req rel diff %d allocated %d\n",
underflow, overflow, req_rel_diff, allocated);
if (overflow) {
p += sprintf(p, "ERROR: overflow counter must be zero\n");
}
if (underflow) {
p += sprintf(p, "WARNING: underflow counter is non zero, may need to increase pool\n");
}
for (pool = 0; pool < TOPAZ_HBM_POOL_COUNT; ++pool) {
p += sprintf(p, "pool %d req rel diff %d, available %u\n", pool, req_rel_perpool_diff[pool],
topaz_hbm_pool_available(pool));
}
#else /* ruby sbm */
int i;
for (i = 0; i < ARRAY_SIZE(sbm_pools); i++) {
struct sbm_pool *pool = &sbm_pools[i];
p += sprintf(p, "sbm pool %d readp %u writep %u count %lu ptr pool %p\n",
i, pool->read, pool->write, BIT(pool->ptr_s), pool->ptrs);
}
#endif
*eof = 1;
return p - page;
}
static int __init topaz_hbm_stat_init(void)
{
if (!create_proc_read_entry(TOPAZ_HBM_PROC_FILENAME, 0,
NULL, topaz_hbm_stat_rd, NULL)) {
return -EEXIST;
}
return 0;
}
static void __exit topaz_hbm_stat_exit(void)
{
remove_proc_entry(TOPAZ_HBM_PROC_FILENAME, 0);
}
void topaz_hbm_init_pool_list(unsigned long *const pool_list, const uint16_t payload_count_s,
const uintptr_t payloads_bus, const uint32_t payload_size,
const uint32_t payload_headroom, const int8_t pool)
{
uint32_t i;
const uint16_t payload_count = (1 << payload_count_s);
topaz_hbm_init((void *) virt_to_bus(pool_list), payload_count_s, pool, 0);
if (payloads_bus) {
for (i = 0; i < payload_count; i++) {
uintptr_t buf_bus = payloads_bus + (i * payload_size) + payload_headroom;
uint32_t *_p = bus_to_virt(buf_bus);
uint32_t *_m = topaz_hbm_buf_get_meta(_p);
uint32_t *enqueuep = _m - HBM_HR_OFFSET_ENQ_CNT;
uint32_t *freep = _m - HBM_HR_OFFSET_FREE_CNT;
uint32_t *statep = _m - HBM_HR_OFFSET_STATE;
uint32_t *magicp = _p - HBM_HR_OFFSET_MAGIC;
uint32_t *guardp =(uint32_t*)((uint32_t)_p + payload_size - payload_headroom -
TOPAZ_HBM_PAYLOAD_END_GUARD_SIZE);
#if TOPAZ_HBM_BUF_EXTERNAL_META
uint32_t *meta_ptr_p = _p - HBM_HR_OFFSET_META_PTR;
uint32_t *meta_backptr_p = _m - HBM_HR_OFFSET_META_PTR;
#endif
int j;
#if TOPAZ_HBM_DEBUG_STAMPS
uint32_t *jiffp = _m - HBM_HR_OFFSET_FREE_JIFF;
uint32_t *ownerp = _m - HBM_HR_OFFSET_OWNER;
arc_write_uncached_32(jiffp, jiffies);
arc_write_uncached_32(ownerp, TOPAZ_HBM_OWNER_INIT);
#endif
/* always setup magic and guard area to provide minimum detection */
arc_write_uncached_32(magicp, TOPAZ_HBM_BUF_GUARD_MAGIC);
arc_write_uncached_32(statep, 0);
for (j = 0; j < (TOPAZ_HBM_PAYLOAD_END_GUARD_SIZE >> 2); j++) {
arc_write_uncached_32((guardp + j), TOPAZ_HBM_BUF_GUARD_MAGIC);
}
arc_write_uncached_32(enqueuep, 1);
arc_write_uncached_32(freep, 0);
#if TOPAZ_HBM_BUF_EXTERNAL_META
arc_write_uncached_32(meta_ptr_p, virt_to_bus(_m));
arc_write_uncached_32(meta_backptr_p, buf_bus);
#endif
topaz_hbm_put_buf((void *) buf_bus, pool);
}
}
printk(KERN_INFO "%s pool %u pool_list 0x%p bus_range 0x%lx to 0x%lx sz %u count %u\n",
__FUNCTION__, pool, pool_list,
payloads_bus, payloads_bus + payload_size * payload_count,
payload_size, payload_count);
}
#ifndef TOPAZ_PLATFORM
struct sbm_pool sbm_pools[4];
#endif
static int g_pools_inited = 0;
static void topaz_hbm_init_payload_pools(void)
{
unsigned long flags;
uintptr_t *topaz_hbm_emac_rx_ptrs = (void *) (RUBY_SRAM_BEGIN + TOPAZ_HBM_POOL_EMAC_RX_START);
uintptr_t *topaz_hbm_wmac_rx_ptrs = (void *) (RUBY_SRAM_BEGIN + TOPAZ_HBM_POOL_WMAC_RX_START);
uintptr_t *topaz_hbm_emac_free_ptrs = (void *) (RUBY_SRAM_BEGIN + TOPAZ_HBM_POOL_EMAC_TX_DONE_START);
printk("HBM pool: emac rx 0x%x to 0x%x, wmac rx 0x%x to 0x%x\n",
TOPAZ_HBM_POOL_EMAC_RX_START,
TOPAZ_HBM_POOL_EMAC_RX_END,
TOPAZ_HBM_POOL_WMAC_RX_START,
TOPAZ_HBM_POOL_WMAC_RX_END);
memset((void *) (RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_EMAC_RX_BASE), TOPAZ_HBM_BUF_PAYLOAD_POISON,
TOPAZ_HBM_BUF_EMAC_RX_TOTAL);
memset((void *) (RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_WMAC_RX_BASE), TOPAZ_HBM_BUF_PAYLOAD_POISON,
TOPAZ_HBM_BUF_WMAC_RX_TOTAL);
flush_and_inv_dcache_sizerange_safe((void *) (RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_EMAC_RX_BASE), TOPAZ_HBM_BUF_EMAC_RX_TOTAL);
flush_and_inv_dcache_sizerange_safe((void *) (RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_WMAC_RX_BASE), TOPAZ_HBM_BUF_WMAC_RX_TOTAL);
memset((void *) (RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_META_BASE), TOPAZ_HBM_BUF_PAYLOAD_POISON,
TOPAZ_HBM_BUF_META_TOTAL);
flush_and_inv_dcache_sizerange_safe((void *)(RUBY_DRAM_BEGIN + TOPAZ_HBM_BUF_META_BASE),
TOPAZ_HBM_BUF_META_TOTAL);
#if TOPAZ_HBM_BUF_EXTERNAL_META
printk("HBM meta: emac rx 0x%x to 0x%x, wmac rx 0x%x to 0x%x\n",
TOPAZ_HBM_BUF_META_EMAC_RX_BASE,
TOPAZ_HBM_BUF_META_EMAC_RX_END,
TOPAZ_HBM_BUF_META_WMAC_RX_BASE,
TOPAZ_HBM_BUF_META_WMAC_RX_END);
#else
printk("HBM used internal meta\n");
#endif
local_irq_save(flags);
topaz_hbm_init_pool_list(topaz_hbm_emac_rx_ptrs, TOPAZ_HBM_BUF_EMAC_RX_COUNT_S,
TOPAZ_HBM_BUF_EMAC_RX_BASE, TOPAZ_HBM_BUF_EMAC_RX_SIZE, TOPAZ_HBM_PAYLOAD_HEADROOM,
TOPAZ_HBM_BUF_EMAC_RX_POOL);
topaz_hbm_init_pool_list(topaz_hbm_wmac_rx_ptrs, TOPAZ_HBM_BUF_WMAC_RX_COUNT_S,
TOPAZ_HBM_BUF_WMAC_RX_BASE, TOPAZ_HBM_BUF_WMAC_RX_SIZE, TOPAZ_HBM_PAYLOAD_HEADROOM,
TOPAZ_HBM_BUF_WMAC_RX_POOL);
topaz_hbm_init_pool_list(topaz_hbm_emac_free_ptrs, TOPAZ_HBM_EMAC_TX_DONE_COUNT_S,
0, 0, 0, TOPAZ_HBM_EMAC_TX_DONE_POOL);
local_irq_restore(flags);
g_pools_inited = 1;
}
int topaz_hbm_handle_buf_err(void *const buf_virt, int8_t dest_pool)
{
if (!topaz_hbm_buf_ptr_valid(buf_virt)) {
HBM_STATS(HBM_CNT_MISALIGNED, 1);
printk(KERN_CRIT "%s: buf 0x%x misaligned: pool %u\n",
__FUNCTION__,
(unsigned int)buf_virt, dest_pool);
return 0;
}
HBM_STATS(HBM_CNT_MAGIC_CORRUPTED, 1);
hbm_buf_fix_buf_magic(buf_virt);
return 1;
}
void __attribute__((section(".sram.text"))) topaz_hbm_filter_txdone_buf(void *const buf_bus)
{
const int8_t dest_pool = topaz_hbm_payload_get_pool_bus(buf_bus);
if (dest_pool == TOPAZ_HBM_BUF_WMAC_RX_POOL ||
dest_pool == TOPAZ_HBM_BUF_EMAC_RX_POOL) {
uint32_t *const _p = bus_to_virt((uintptr_t) buf_bus);
uint32_t *_m = topaz_hbm_buf_get_meta(_p);
uint32_t *const enqueuep = _m - HBM_HR_OFFSET_ENQ_CNT;
uint32_t *const freep = _m - HBM_HR_OFFSET_FREE_CNT;
const uint32_t ec = arc_read_uncached_32(enqueuep);
const uint32_t fc = arc_read_uncached_32(freep) + 1;
const bool release = (ec && (fc == ec));
if (release) {
/* only fix magic corruption when we are sure no one else is using it right now */
if (TOPAZ_HBM_BUF_MAGIC_CHK_ALLPOOL || (dest_pool == TOPAZ_HBM_BUF_WMAC_RX_POOL)) {
int state = hbm_buf_check_buf_magic(_p);
if (unlikely(state)) {
if (!topaz_hbm_handle_buf_err(_p, dest_pool)) {
/* shouldn't put it back to pool */
return;
}
}
}
#if TOPAZ_HBM_DEBUG_STAMPS
uint32_t *jiffp = _m - HBM_HR_OFFSET_FREE_JIFF;
uint32_t *ownerp = _m - HBM_HR_OFFSET_OWNER;
const uint32_t owner = arc_read_uncached_32(ownerp);
const uint8_t owner1 = (owner & 0xF) >> 0;
if (owner1 == TOPAZ_HBM_OWNER_FREE) {
/*
* Double free check is already broken because a lot of free places
* doesn't update the owner, both VB and PCIe platform.
*/
/*
uint32_t *sizep = _m - HBM_HR_OFFSET_SIZE;
const uint32_t size = arc_read_uncached_32(sizep);
printk(KERN_ERR "%s: double free of buf_bus %p size %u owner %08x\n",
__FUNCTION__, buf_bus, size, owner);
topaz_hbm_buf_show(_p, TOPAZ_HBM_BUF_DUMP_DFT, 0);
*/
}
arc_write_uncached_32(jiffp, jiffies);
arc_write_uncached_32(ownerp, (owner << 4) | TOPAZ_HBM_OWNER_FREE);
#endif
if (ec != 1) {
arc_write_uncached_32(enqueuep, 1);
arc_write_uncached_32(freep, 0);
}
topaz_hbm_put_payload_aligned_bus(buf_bus, dest_pool);
} else {
arc_write_uncached_32(freep, fc);
}
} else {
HBM_STATS(HBM_CNT_INVALID_POOL, 1);
printk(KERN_CRIT "%s: unknown pool %hhd for buf_bus 0x%p\n",
__FUNCTION__, dest_pool, buf_bus);
}
}
EXPORT_SYMBOL(topaz_hbm_filter_txdone_buf);
/*
* Safely return the buf.
* @pkt_bus needn't to be the pointer from the pool. It can be any location in the buffer.
*/
void topaz_hbm_release_buf_safe(void *const pkt_bus)
{
const int8_t dest_pool = topaz_hbm_payload_get_pool_bus(pkt_bus);
void *buf_bus = topaz_hbm_payload_store_align_bus(pkt_bus, dest_pool, 0);
unsigned long flags;
local_irq_save(flags);
topaz_hbm_filter_txdone_buf(buf_bus);
local_irq_restore(flags);
}
EXPORT_SYMBOL(topaz_hbm_release_buf_safe);
void topaz_hbm_filter_txdone_pool(void)
{
unsigned long flags;
void *buf_bus;
const int8_t src_pool = TOPAZ_HBM_EMAC_TX_DONE_POOL;
const uint32_t mask = TOPAZ_HBM_EMAC_TX_DONE_COUNT - 1;
uint32_t wr_ptr;
uint32_t rd_ptr;
uint32_t wr_raw;
uint32_t rd_raw;
uint32_t i;
uint32_t count;
uint32_t full;
if (unlikely(!g_pools_inited)) {
return;
}
local_irq_save(flags);
wr_raw = readl(TOPAZ_HBM_WR_PTR(src_pool));
rd_raw = readl(TOPAZ_HBM_RD_PTR(src_pool));
wr_ptr = wr_raw & mask;
rd_ptr = rd_raw & mask;
full = ((wr_raw != rd_raw) && (wr_ptr == rd_ptr));
for (count = 0, i = rd_ptr; ((i != wr_ptr) || full); i = (i + 1) & mask, count++) {
buf_bus = topaz_hbm_get_payload_bus(src_pool);
if (buf_bus != NULL) {
topaz_hbm_filter_txdone_buf(buf_bus);
} else if (printk_ratelimit()) {
printk(KERN_CRIT "%s: read NULL from pool %d\n",
__FUNCTION__, src_pool);
break;
}
full = 0;
}
#ifdef TOPAZ_EMAC_NULL_BUF_WR
if (topaz_emac_null_buf_del_cb) {
uint32_t n;
wr_ptr = readl(TOPAZ_HBM_WR_PTR(TOPAZ_HBM_BUF_EMAC_RX_POOL));
rd_ptr = readl(TOPAZ_HBM_RD_PTR(TOPAZ_HBM_BUF_EMAC_RX_POOL));
n = (wr_ptr - rd_ptr) % TOPAZ_HBM_BUF_EMAC_RX_COUNT;
if (n > HBM_UFLOW_RECOVER_TH)
topaz_emac_null_buf_del_cb();
}
#endif
local_irq_restore(flags);
if (unlikely(count > (TOPAZ_HBM_EMAC_TX_DONE_COUNT * 3 / 4))) {
if (printk_ratelimit())
printk("Warning! %s count: %u\n", __FUNCTION__, count);
}
}
EXPORT_SYMBOL(topaz_hbm_filter_txdone_pool);
static struct kmem_cache *shinfo_cache;
static uint8_t *topaz_hbm_skb_allocator_payload_alloc(struct skb_shared_info **shinfo,
size_t size, gfp_t gfp_mask, int node)
{
uint8_t *data;
size = SKB_DATA_ALIGN(size);
*shinfo = kmem_cache_alloc(shinfo_cache, gfp_mask);
if (*shinfo == NULL) {
return NULL;
}
if (size < topaz_hbm_pool_buf_max_size(TOPAZ_HBM_BUF_EMAC_RX_POOL)) {
data = topaz_hbm_get_payload_virt(TOPAZ_HBM_BUF_EMAC_RX_POOL);
} else {
data = kmalloc(size, gfp_mask);
}
if (data == NULL) {
kmem_cache_free(shinfo_cache, *shinfo);
*shinfo = NULL;
}
return data;
}
static void topaz_hbm_skb_allocator_payload_free(struct sk_buff *skb)
{
void *buf_bus = (void *) virt_to_bus(skb->head);
const int8_t pool = topaz_hbm_payload_get_free_pool_bus(buf_bus);
kmem_cache_free(shinfo_cache, skb_shinfo(skb));
if (topaz_hbm_pool_valid(pool)) {
if (!skb->hbm_no_free) {
unsigned long flags;
local_irq_save(flags);
topaz_hbm_flush_skb_cache(skb);
topaz_hbm_filter_txdone_buf(buf_bus);
local_irq_restore(flags);
}
} else {
kfree(skb->head);
}
topaz_hbm_filter_txdone_pool();
}
const struct skb_allocator topaz_hbm_skb_allocator = {
.name = "topaz_hbm",
.skb_alloc = &skb_allocator_kmem_caches_skb_alloc,
.skb_free = &skb_allocator_kmem_caches_skb_free,
.payload_alloc = &topaz_hbm_skb_allocator_payload_alloc,
.payload_free = &topaz_hbm_skb_allocator_payload_free,
.max_size = 0,
};
#define QTN_HBM_MAX_FRAME_LEN 12000 /* 12000 is over maximum vht frame size,
and there is no rx frame whose size is over 12000 */
struct sk_buff *_topaz_hbm_attach_skb(void *buf_virt, int8_t pool
QTN_SKB_ALLOC_TRACE_ARGS)
{
struct sk_buff *skb;
struct skb_shared_info *shinfo;
uint32_t buf_size;
uint8_t *buf_head;
shinfo = kmem_cache_alloc(shinfo_cache, GFP_ATOMIC);
if (!shinfo) {
return NULL;
}
skb = skb_allocator_kmem_caches_skb_alloc(GFP_ATOMIC, 0, -1);
if (!skb) {
kmem_cache_free(shinfo_cache, shinfo);
return NULL;
}
/*
* TODO FIXME: Restrict the buffer size less than 12k, because we saw ping failed
* if we set skb buffer size as 17K
*/
buf_size = min((int)topaz_hbm_pool_buf_max_size(pool), QTN_HBM_MAX_FRAME_LEN);
buf_head = topaz_hbm_payload_store_align_virt(buf_virt, pool, 0);
/* invalidate all packet dcache before passing to the kernel */
inv_dcache_sizerange_safe(buf_head, buf_size);
__alloc_skb_init(skb, shinfo, buf_head,
buf_size, 0, &topaz_hbm_skb_allocator
QTN_SKB_ALLOC_TRACE_ARGVARS);
skb_reserve(skb, ((uint8_t *) buf_virt) - buf_head);
return skb;
}
EXPORT_SYMBOL(_topaz_hbm_attach_skb);
/*
* Allocate a new buffer to hold the pkt. The new buffer is guranteed to be safe from wmac rx dma overrun.
* The original buffer is not used anymore. Caller should be responsible for freeing the buffer.
*/
struct sk_buff *topaz_hbm_attach_skb_quarantine(void *buf_virt, int pool, int len, uint8_t **whole_frm_hdr_p)
{
uint8_t *buf_head;
struct sk_buff *skb = NULL;
uint32_t prev_len;
uint32_t buf_size;
KASSERT((pool == TOPAZ_HBM_BUF_WMAC_RX_POOL), ("buf quarantine is only for wmac rx pool, %d", pool));
buf_head = topaz_hbm_payload_store_align_virt(buf_virt, pool, 0);
if (hbm_buf_check_buf_magic(buf_head)) {
/* don't copy to new skb if it is aleady corrupted */
HBM_STATS(HBM_CNT_QUARANTINE_CORRUPTED, 1);
return NULL;
}
/* copy from buffer head in case mac header is needed */
prev_len = (uint32_t)buf_virt - (uint32_t)buf_head;
buf_size = prev_len + len;
skb = dev_alloc_skb(buf_size);
if (!skb) {
HBM_STATS(HBM_CNT_QUARANTINE_ALLOC_FAIL, 1);
return NULL;
}
/* caller only invalidate this pkt, not entire buffer */
inv_dcache_sizerange_safe(buf_head, prev_len);
if (whole_frm_hdr_p)
*whole_frm_hdr_p = skb->data;
memcpy(skb->data, buf_head, buf_size);
if (hbm_buf_check_buf_magic(buf_head)) {
/* if corruption happens during data copying */
HBM_STATS(HBM_CNT_QUARANTINE_CORRUPTED, 1);
goto post_check_fail;
}
/* reserve head space so that later caller's skb_put() covers the packet */
skb_reserve(skb, prev_len);
HBM_STATS(HBM_CNT_QUARANTINE_OK, 1);
/*
* Quarantine is done. now we are sure the data copy in skb is not corrupted and won't
*/
return skb;
post_check_fail:
if (skb)
dev_kfree_skb(skb);
return NULL;
}
EXPORT_SYMBOL(topaz_hbm_attach_skb_quarantine);
static int topaz_hbm_bufs_sort_need_swap(const uint32_t *buf0, const uint32_t *buf1)
{
int type;
int inv;
uint32_t v0;
uint32_t v1;
type = topaz_hbm_dump_sort_type;
inv = 0;
if (topaz_hbm_dump_sort_type >= HBM_DUMP_SORT_ORDER_INV_BASE) {
type -= HBM_DUMP_SORT_ORDER_INV_BASE;
inv = 1;
}
switch(type) {
case HBM_DUMP_SORT_ADDR:
v0 = (uint32_t)buf0;
v1 = (uint32_t)buf1;
break;
case HBM_DUMP_SORT_JIFF:
v0 = jiffies - arc_read_uncached_32(topaz_hbm_buf_get_meta(buf0) - HBM_HR_OFFSET_FREE_JIFF);
v1 = jiffies - arc_read_uncached_32(topaz_hbm_buf_get_meta(buf1) - HBM_HR_OFFSET_FREE_JIFF);
break;
case HBM_DUMP_SORT_BAD_MAGIC:
v0 = (arc_read_uncached_32(buf0 - HBM_HR_OFFSET_MAGIC) == TOPAZ_HBM_BUF_GUARD_MAGIC);
v1 = (arc_read_uncached_32(buf1 - HBM_HR_OFFSET_MAGIC) == TOPAZ_HBM_BUF_GUARD_MAGIC);
break;
default:
return 0;
break;
}
return (inv ? (v1 > v0) : (v0 > v1));
}
static void topaz_hbm_bufs_sort(int pool, int pool_size)
{
int i;
int j;
uint32_t *buf0;
uint32_t *buf1;
uint32_t *buf;
int swapped;
memset(topaz_hbm_dump_bufs_sorted, 0, sizeof(topaz_hbm_dump_bufs_sorted));
for (i = 0; i < pool_size; i++) {
topaz_hbm_dump_bufs_sorted[i] = (uint32_t*)
topaz_hbm_payload_store_align_from_index(pool, i);
}
/* bubble sort */
for (i = 0; i < (pool_size - 1); i++) {
swapped = 0;
for (j = 0; j < (pool_size - i - 1); j++) {
buf0 = topaz_hbm_dump_bufs_sorted[j];
buf1 = topaz_hbm_dump_bufs_sorted[j + 1];
if (topaz_hbm_bufs_sort_need_swap(buf0, buf1)) {
buf = buf0;
topaz_hbm_dump_bufs_sorted[i] = buf1;
topaz_hbm_dump_bufs_sorted[j] = buf;
swapped = 1;
}
}
if (!swapped)
break;
}
topaz_hbm_dumped_num = 0;
}
static int topaz_hbm_buf_in_range(const uint32_t *buf)
{
int type;
uint32_t v;
type = topaz_hbm_dump_sort_type;
if (topaz_hbm_dump_sort_type >= HBM_DUMP_SORT_ORDER_INV_BASE) {
type -= HBM_DUMP_SORT_ORDER_INV_BASE;
}
switch(type) {
case HBM_DUMP_SORT_ADDR:
v = (uint32_t)buf;
break;
case HBM_DUMP_SORT_JIFF:
v = jiffies - arc_read_uncached_32(topaz_hbm_buf_get_meta(buf) - HBM_HR_OFFSET_FREE_JIFF);
break;
case HBM_DUMP_SORT_BAD_MAGIC:
v = (arc_read_uncached_32(buf - HBM_HR_OFFSET_MAGIC) == TOPAZ_HBM_BUF_GUARD_MAGIC);
break;
default:
return 0;
break;
}
return ((topaz_hbm_dump_sort_range_min <= v) && (v <= topaz_hbm_dump_sort_range_max));
}
static void *topaz_hbm_bufs_emac_seq_start(struct seq_file *sfile, loff_t *pos)
{
if (*pos > (TOPAZ_HBM_POOL_SIZE_MAX - 1))
return NULL;
if (*pos == 0)
topaz_hbm_bufs_sort(TOPAZ_HBM_BUF_EMAC_RX_POOL, TOPAZ_HBM_BUF_EMAC_RX_COUNT);
return topaz_hbm_dump_bufs_sorted[*pos];
}
static void *topaz_hbm_bufs_wmac_seq_start(struct seq_file *sfile, loff_t *pos)
{
if (*pos > (TOPAZ_HBM_POOL_SIZE_MAX - 1))
return NULL;
if (*pos == 0)
topaz_hbm_bufs_sort(TOPAZ_HBM_BUF_WMAC_RX_POOL, TOPAZ_HBM_BUF_WMAC_RX_COUNT);
return topaz_hbm_dump_bufs_sorted[*pos];
}
static void* topaz_hbm_bufs_seq_next(struct seq_file *sfile, void *v, loff_t *pos)
{
if (*pos > (TOPAZ_HBM_POOL_SIZE_MAX - 1))
return NULL;
*pos += 1;
return topaz_hbm_dump_bufs_sorted[*pos];
}
static void topaz_hbm_bufs_seq_stop(struct seq_file *sfile, void *v)
{
}
static int topaz_hbm_bufs_seq_show(struct seq_file *sfile, void *v)
{
const uint32_t *_p = v;
const uint32_t *_m = topaz_hbm_buf_get_meta(_p);
const uint32_t *enqueuep = _m - HBM_HR_OFFSET_ENQ_CNT;
const uint32_t *freep = _m - HBM_HR_OFFSET_FREE_CNT;
const uint32_t *jiffp = _m - HBM_HR_OFFSET_FREE_JIFF;
const uint32_t *ownerp = _m - HBM_HR_OFFSET_OWNER;
const uint32_t *sizep = _m - HBM_HR_OFFSET_SIZE;
const uint32_t *magicp = _p - HBM_HR_OFFSET_MAGIC;
const uint32_t ec = arc_read_uncached_32(enqueuep);
const uint32_t fc = arc_read_uncached_32(freep);
const uint32_t jc = arc_read_uncached_32(jiffp);
const uint32_t oc = arc_read_uncached_32(ownerp);
const uint32_t sz = arc_read_uncached_32(sizep);
const uint32_t magic = arc_read_uncached_32(magicp);
const uint8_t *d;
int dump_bytes;
int i;
uint8_t *tail;
int tail_bytes;
uint32_t whole_size;
uint32_t payload_size;
int pool;
uint32_t idx;
if (!topaz_hbm_buf_in_range(_p)) {
return 0;
}
if (topaz_hbm_dumped_num++ >= topaz_hbm_dump_num) {
return 0;
}
pool = topaz_hbm_buf_identify_buf_virt(v, &whole_size, &idx);
if (pool < 0) {
seq_printf(sfile, "invalid hbm buffer %x\n", (unsigned int)v);
return 0;
}
payload_size = whole_size - TOPAZ_HBM_PAYLOAD_HEADROOM;
dump_bytes = (topaz_hbm_dump_len == TOPAZ_HBM_BUF_DUMP_MAX) ? payload_size : topaz_hbm_dump_len;
d = v;
inv_dcache_sizerange_safe(v, dump_bytes);
seq_printf(sfile, "%p ec %u fp %u own %08x size %u j %u (%u s ago) mg %x\n",
v, ec, fc, oc, sz, jc, (((uint32_t) jiffies) - jc) / HZ, magic);
for (i = 0; i < dump_bytes; ) {
if (!(i % 32))
seq_printf(sfile, "%08x ", (i - i % 32));
++i;
seq_printf(sfile, "%02x%s", *d++, (i % 32) == 0 ? "\n" : " ");
}
if (topaz_hbm_dump_taillen) {
seq_printf(sfile, "\n");
tail_bytes = topaz_hbm_dump_taillen;
tail = (uint8_t*)((uint32_t)v + payload_size - tail_bytes);
inv_dcache_sizerange_safe(tail, tail_bytes);
seq_printf(sfile, "%p tail %p\n", v, tail);
for (i = 0; i < tail_bytes; ) {
if (!(i % 32))
seq_printf(sfile, "%08x ", (i - i % 32));
++i;
seq_printf(sfile, "%02x%s", *tail++, (i % 32) == 0 ? "\n" : " ");
}
}
seq_printf(sfile, "\n");
return 0;
}
static struct seq_operations topaz_hbm_bufs_emac_seq_ops = {
.start = topaz_hbm_bufs_emac_seq_start,
.next = topaz_hbm_bufs_seq_next,
.stop = topaz_hbm_bufs_seq_stop,
.show = topaz_hbm_bufs_seq_show
};
static struct seq_operations topaz_hbm_bufs_wmac_seq_ops = {
.start = topaz_hbm_bufs_wmac_seq_start,
.next = topaz_hbm_bufs_seq_next,
.stop = topaz_hbm_bufs_seq_stop,
.show = topaz_hbm_bufs_seq_show
};
static int topaz_hbm_bufs_emac_proc_open(struct inode *inode, struct file *file)
{
return seq_open(file, &topaz_hbm_bufs_emac_seq_ops);
}
static int topaz_hbm_bufs_wmac_proc_open(struct inode *inode, struct file *file)
{
return seq_open(file, &topaz_hbm_bufs_wmac_seq_ops);
}
static struct file_operations topaz_hbm_bufs_emac_proc_ops = {
.owner = THIS_MODULE,
.open = topaz_hbm_bufs_emac_proc_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release
};
static struct file_operations topaz_hbm_bufs_wmac_proc_ops = {
.owner = THIS_MODULE,
.open = topaz_hbm_bufs_wmac_proc_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release
};
static inline int hbm_if_split_words(char **words, char *str)
{
int word_count = 0;
/* skip leading space */
while (str && *str && isspace(*str)) {
str++;
}
while (str && *str) {
words[word_count++] = str;
/* skip this word */
while (str && *str && !isspace(*str)) {
str++;
}
/* replace spaces with NULL */
while (str && *str && isspace(*str)) {
*str = 0;
str++;
}
}
return word_count;
}
static int hbm_if_cmd_dumpctl(char **words, uint8_t word_count)
{
int idx = 1;
if (word_count >= (idx + 1))
sscanf(words[idx++], "%u", &topaz_hbm_dump_sort_type);
if (word_count >= (idx + 1))
sscanf(words[idx++], "0x%x", &topaz_hbm_dump_sort_range_min);
if (word_count >= (idx + 1))
sscanf(words[idx++], "0x%x", &topaz_hbm_dump_sort_range_max);
if (word_count >= (idx + 1))
sscanf(words[idx++], "%u", &topaz_hbm_dump_num);
if (word_count >= (idx + 1))
sscanf(words[idx++], "%u", &topaz_hbm_dump_len);
if (word_count >= (idx + 1))
sscanf(words[idx++], "%u", &topaz_hbm_dump_taillen);
printk("hbm_if set dump ctl: sort_type %u sort_range [0x%x 0x%x] num %u len %u %u\n",
topaz_hbm_dump_sort_type,
topaz_hbm_dump_sort_range_min,
topaz_hbm_dump_sort_range_max,
topaz_hbm_dump_num,
topaz_hbm_dump_len,
topaz_hbm_dump_taillen
);
return 0;
}
static int hbm_if_cmd_show_stats(char **words, uint8_t word_count)
{
int i;
printk("HBM stats:\n");
for (i = 0; i < HBM_CNT_NUM; i++) {
printk("%s = %u\n", topaz_hbm_stats_names[i], topaz_hbm_stats[i]);
}
return 0;
}
/* Apply user command.
* User command can control the HBM interface.
* @param cmd_num: command number
* @param words: the split words without spaces from the user space console interface
* @param word_count: number of words after split
* @return: status indication
*/
static int hbm_if_apply_user_command(hbm_if_usr_cmd cmd_num, char **words, uint8_t word_count)
{
int rc = -EINVAL;
if ((word_count == 0) || (!words)) {
goto cmd_failure;
}
switch(cmd_num) {
case HBM_IF_CMD_DUMPCTL:
rc = hbm_if_cmd_dumpctl(words, word_count);
break;
case HBM_IF_CMD_STATS:
rc = hbm_if_cmd_show_stats(words, word_count);
break;
default:
goto cmd_failure;
break;
}
if (rc < 0) {
goto cmd_failure;
}
return 1;
cmd_failure:
if (words)
printk(KERN_INFO "Failed to parse command:%s, word count:%d\n", *words, word_count);
else
printk(KERN_INFO "Failed to parse command:(NULL)\n");
return -EPERM;
}
static int hbm_if_write_proc(struct file *file, const char __user *buffer,
unsigned long count, void *_unused)
{
char *cmd;
int rc, i;
char **words;
uint8_t word_count;
hbm_if_usr_cmd cmd_num = 0;
cmd = kmalloc(count, GFP_KERNEL);
words = kmalloc(count * sizeof(char *) / 2, GFP_KERNEL);
if (!cmd || !words) {
rc = -ENOMEM;
goto out;
}
if (copy_from_user(cmd, buffer, count)) {
rc = -EFAULT;
goto out;
}
/* Set null at last byte, note that count already gives +1 byte count*/
cmd[count - 1] = '\0';
word_count = hbm_if_split_words(words, cmd);
for (i = 0; i < HBM_IF_MAX_CMD; i++, cmd_num++) {
/* Extract command from first word */
if (strcmp(words[0], str_cmd[i]) == 0) {
printk(KERN_INFO"HBM user command:%s \n", str_cmd[i]);
break;
}
}
/* Exclude softirqs whilst manipulating forwarding table */
local_bh_disable();
rc = hbm_if_apply_user_command(cmd_num, words, word_count);
local_bh_enable();
out:
if (cmd) {
kfree(cmd);
}
if (words) {
kfree(words);
}
return count;
}
static inline uint32_t hbm_get_rel_cnt(int pool)
{
int master;
uint32_t rel = 0;
for (master = 0; master < TOPAZ_HBM_MASTER_COUNT; ++master)
rel += readl(TOPAZ_HBM_POOL_RELEASE_CNT(master, pool));
return rel;
}
static int topaz_hbm_pool_poll_stat(int pool, struct hbm_pool_cnt *ps)
{
uint32_t free;
uint32_t fdelt;
int rc;
free = hbm_get_rel_cnt(pool);
fdelt = free - ps->prev_release_cnt;
ps->prev_release_cnt = free;
if ((topaz_hbm_pool_available(pool) < HBM_BUF_MINIMUM_AVAIL_NUM) &&
(fdelt < HBM_BUF_MINIMUM_REL_NUM)) {
ps->pool_depleted_cnt++;
rc = -1;
} else {
ps->pool_depleted_cnt = 0;
rc = 0;
}
return rc;
}
void topaz_hbm_monitor(unsigned long data)
{
struct topaz_hbm_mnt *hm = (struct topaz_hbm_mnt *)data;
uint32_t uf;
int rc;
unsigned long intval;
intval = HBM_BUF_POLL_L_INTRVAL;
if (!hm->unflow_flag) {
uf = readl(TOPAZ_HBM_UNDERFLOW_CNT);
if (uf - hm->prev_unflow_cnt) {
hm->unflow_flag = 1;
hm->prev_unflow_cnt = uf;
} else {
goto exit;
}
}
rc = topaz_hbm_pool_poll_stat(TOPAZ_HBM_BUF_WMAC_RX_POOL, &hm->wmac_pl);
rc += topaz_hbm_pool_poll_stat(TOPAZ_HBM_BUF_EMAC_RX_POOL, &hm->emac_pl);
if (rc == 0) {
hm->unflow_flag = 0;
} else {
if ((hm->wmac_pl.pool_depleted_cnt > HBM_BUF_DEPLETION_TH) ||
(hm->wmac_pl.pool_depleted_cnt > HBM_BUF_DEPLETION_TH)) {
panic("HBM pool is depleted, wmac pool:%d, emac rx poll:%d\n",
topaz_hbm_pool_available(TOPAZ_HBM_BUF_WMAC_RX_POOL),
topaz_hbm_pool_available(TOPAZ_HBM_BUF_EMAC_RX_POOL));
}
intval = HBM_BUF_POLL_S_INTRVAL;
}
exit:
mod_timer(&hbm_timer, jiffies + intval);
}
static int __init topaz_hbm_bufs_init(void)
{
struct topaz_hbm_mnt *hm;
struct proc_dir_entry *e;
if ((e = create_proc_entry("hbm_bufs_emac", 0, NULL)) != NULL) {
e->proc_fops = &topaz_hbm_bufs_emac_proc_ops;
}
if ((e = create_proc_entry("hbm_bufs_wmac", 0, NULL)) != NULL) {
e->proc_fops = &topaz_hbm_bufs_wmac_proc_ops;
}
struct proc_dir_entry *entry = create_proc_entry(TOPAZ_HBM_IF_PROC_NAME, 0600, NULL);
if (entry) {
entry->write_proc = hbm_if_write_proc;
entry->read_proc = NULL;
}
hm = (struct topaz_hbm_mnt *)kzalloc(sizeof(*hm), GFP_KERNEL);
if (!hm) {
printk(KERN_ERR"%s: fail to allocate hm", __func__);
return -1;
}
init_timer(&hbm_timer);
hbm_timer.data = (unsigned long)hm;
hbm_timer.function = &topaz_hbm_monitor;
mod_timer(&hbm_timer, jiffies + HBM_BUF_POLL_L_INTRVAL);
return 0;
}
static void __exit topaz_hbm_bufs_exit(void)
{
remove_proc_entry("hbm_bufs_wmac", 0);
remove_proc_entry("hbm_bufs_emac", 0);
remove_proc_entry(TOPAZ_HBM_IF_PROC_NAME, NULL);
del_timer(&hbm_timer);
if (hbm_timer.data)
kfree((void *)hbm_timer.data);
}
static int __init topaz_hbm_module_init(void)
{
COMPILE_TIME_ASSERT(TOPAZ_HBM_BUF_META_SIZE >= (HBM_HR_OFFSET_MAX * 4));
topaz_hbm_init_payload_pools();
shinfo_cache = kmem_cache_create("topaz_skb_shinfo_cache",
sizeof(struct skb_shared_info),
0,
SLAB_HWCACHE_ALIGN | SLAB_PANIC,
NULL);
skb_allocator_register(TOPAZ_HBM_SKB_ALLOCATOR, &topaz_hbm_skb_allocator, 0);
topaz_hbm_stat_init();
topaz_hbm_bufs_init();
return 0;
}
module_init(topaz_hbm_module_init)
static void __exit topaz_hbm_module_exit(void)
{
topaz_hbm_stat_exit();
topaz_hbm_bufs_exit();
}
module_exit(topaz_hbm_module_exit)