Google Git
Sign in
gfiber / vendor / mindspeed / drivers / 8fc946c7f33e98d369655c7c41cc99db6ed9584c / . / pfe_ctrl / pfe_eth.c
blob: d6681ac0b3c0a6a191846dffdc9680165cc09d7b [file] [log] [blame]
/** @pfe_eth.c.
* Ethernet driver for to handle exception path for PFE.
* - uses HIF functions to send/receive packets.
* - uses ctrl function to start/stop interfaces.
* - uses direct register accesses to control phy operation.
*/
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/mii.h>
#include <linux/phy.h>
#include <linux/timer.h>
#include <linux/hrtimer.h>
#include <linux/platform_device.h>
#include <net/ip.h>
#include <net/sock.h>
#include <asm/io.h>
#include <asm/irq.h>
#include <asm/delay.h>
#if defined(CONFIG_NF_CONNTRACK_MARK)
#include <net/netfilter/nf_conntrack.h>
#endif
#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
#include <net/xfrm.h>
#endif
#include "pfe_mod.h"
#include "pfe_eth.h"
const char comcerto_eth_driver_version[]="1.0";
static void *cbus_emac_base[3];
static void *cbus_gpi_base[3];
static void pfe_eth_exit_one(struct pfe_eth_priv_s *priv);
static void pfe_eth_flush_tx(struct pfe_eth_priv_s *priv, int force);
static void pfe_eth_flush_txQ(struct pfe_eth_priv_s *priv, int txQ_num, int from_tx, int n_desc);
#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
extern struct xfrm_state *xfrm_state_lookup_byhandle(struct net *net, u16 handle);
#endif
unsigned int gemac_regs[] = {
0x0000, /* Network control */
0x0004, /* Network configuration */
0x0008, /* Network status */
0x0010, /* DMA configuration */
0x0014, /* Transmit status */
0x0020, /* Receive status */
0x0024, /* Interrupt status */
0x0030, /* Interrupt mask */
0x0038, /* Received pause quantum */
0x003c, /* Transmit pause quantum */
0x0080, /* Hash register bottom [31:0] */
0x0084, /* Hash register bottom [63:32] */
0x0088, /* Specific address 1 bottom [31:0] */
0x008c, /* Specific address 1 top [47:32] */
0x0090, /* Specific address 2 bottom [31:0] */
0x0094, /* Specific address 2 top [47:32] */
0x0098, /* Specific address 3 bottom [31:0] */
0x009c, /* Specific address 3 top [47:32] */
0x00a0, /* Specific address 4 bottom [31:0] */
0x00a4, /* Specific address 4 top [47:32] */
0x00a8, /* Type ID Match 1 */
0x00ac, /* Type ID Match 2 */
0x00b0, /* Type ID Match 3 */
0x00b4, /* Type ID Match 4 */
0x00b8, /* Wake Up ON LAN */
0x00bc, /* IPG stretch register */
0x00c0, /* Stacked VLAN Register */
0x00fc, /* Module ID */
0x07a0 /* EMAC Control register */
};
/********************************************************************/
/* SYSFS INTERFACE */
/********************************************************************/
#if defined(CONFIG_SMP) && (NR_CPUS > 1)
/** pfe_eth_show_rx_cpu_affinity
*
*/
static ssize_t pfe_eth_show_rx_cpu_affinity(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
return sprintf(buf, "%d\n", priv->cpu_id);
}
/** pfe_eth_set_rx_cpu_affinity
*
*/
static ssize_t pfe_eth_set_rx_cpu_affinity(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
unsigned int cpu_id = 0;
sscanf(buf, "%d", &cpu_id);
if (cpu_id < NR_CPUS) {
priv->cpu_id = (int)cpu_id;
hif_lib_set_rx_cpu_affinity(&priv->client, priv->cpu_id);
}
else
printk(KERN_ERR "%s: Invalid CPU (%d)\n", __func__, cpu_id);
return count;
}
#endif
#ifdef PFE_ETH_TSO_STATS
/** pfe_eth_show_tso_stats
*
*/
static ssize_t pfe_eth_show_tso_stats(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
ssize_t len = 0;
int i;
for (i = 0; i < 32; i++)
len += sprintf(buf + len, "TSO packets > %dKBytes = %u\n", i * 2, priv->tso.len_counters[i]);
return len;
}
/** pfe_eth_set_tso_stats
*
*/
static ssize_t pfe_eth_set_tso_stats(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
memset(priv->tso.len_counters, 0, sizeof(priv->tso.len_counters));
return count;
}
#endif
#ifdef PFE_ETH_LRO_STATS
/*
* pfe_eth_show_lro_nb_stats
*/
static ssize_t pfe_eth_show_lro_nb_stats(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
ssize_t len = 0;
int i;
for (i = 0; i < LRO_NB_COUNT_MAX; i++)
len += sprintf(buf + len, "%d fragments packets = %u\n", i, priv->lro_nb_counters[i]);
return len;
}
/*
* pfe_eth_set_lro_nb_stats
*/
static ssize_t pfe_eth_set_lro_nb_stats(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
memset(priv->lro_nb_counters, 0, sizeof(priv->lro_nb_counters));
return count;
}
/*
* pfe_eth_show_lro_len_stats
*/
static ssize_t pfe_eth_show_lro_len_stats(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
ssize_t len = 0;
int i;
for (i = 0; i < LRO_LEN_COUNT_MAX; i++)
len += sprintf(buf + len, "RX packets > %dKBytes = %u\n", i * 2, priv->lro_len_counters[i]);
return len;
}
/*
* pfe_eth_set_lro_len_stats
*/
static ssize_t pfe_eth_set_lro_len_stats(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
memset(priv->lro_len_counters, 0, sizeof(priv->lro_len_counters));
return count;
}
#endif
#ifdef PFE_ETH_NAPI_STATS
/*
* pfe_eth_show_napi_stats
*/
static ssize_t pfe_eth_show_napi_stats(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
ssize_t len = 0;
len += sprintf(buf + len, "sched: %u\n", priv->napi_counters[NAPI_SCHED_COUNT]);
len += sprintf(buf + len, "poll: %u\n", priv->napi_counters[NAPI_POLL_COUNT]);
len += sprintf(buf + len, "packet: %u\n", priv->napi_counters[NAPI_PACKET_COUNT]);
len += sprintf(buf + len, "budget: %u\n", priv->napi_counters[NAPI_FULL_BUDGET_COUNT]);
len += sprintf(buf + len, "desc: %u\n", priv->napi_counters[NAPI_DESC_COUNT]);
return len;
}
/*
* pfe_eth_set_napi_stats
*/
static ssize_t pfe_eth_set_napi_stats(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
memset(priv->napi_counters, 0, sizeof(priv->napi_counters));
return count;
}
#endif
#ifdef PFE_ETH_TX_STATS
/** pfe_eth_show_tx_stats
*
*/
static ssize_t pfe_eth_show_tx_stats(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
ssize_t len = 0;
int i;
len += sprintf(buf + len, "TX queues stats:\n");
for (i = 0; i < EMAC_TXQ_CNT; i++) {
struct netdev_queue *tx_queue = netdev_get_tx_queue(priv->dev, i);
len += sprintf(buf + len, "\n");
__netif_tx_lock_bh(tx_queue);
hif_tx_lock(&pfe->hif);
len += sprintf(buf + len, "Queue %2d : credits = %10d\n", i, hif_lib_tx_credit_avail(pfe, priv->id, i));
len += sprintf(buf + len, " tx packets = %10d\n", pfe->tmu_credit.tx_packets[priv->id][i]);
hif_tx_unlock(&pfe->hif);
/* Don't output additionnal stats if queue never used */
if (!pfe->tmu_credit.tx_packets[priv->id][i])
goto skip;
len += sprintf(buf + len, " clean_fail = %10d\n", priv->clean_fail[i]);
len += sprintf(buf + len, " stop_queue = %10d\n", priv->stop_queue_total[i]);
len += sprintf(buf + len, " stop_queue_hif = %10d\n", priv->stop_queue_hif[i]);
len += sprintf(buf + len, " stop_queue_hif_client = %10d\n", priv->stop_queue_hif_client[i]);
len += sprintf(buf + len, " stop_queue_credit = %10d\n", priv->stop_queue_credit[i]);
skip:
__netif_tx_unlock_bh(tx_queue);
}
return len;
}
/** pfe_eth_set_tx_stats
*
*/
static ssize_t pfe_eth_set_tx_stats(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
int i;
for (i = 0; i < EMAC_TXQ_CNT; i++) {
struct netdev_queue *tx_queue = netdev_get_tx_queue(priv->dev, i);
__netif_tx_lock_bh(tx_queue);
priv->clean_fail[i] = 0;
priv->stop_queue_total[i] = 0;
priv->stop_queue_hif[i] = 0;
priv->stop_queue_hif_client[i]= 0;
priv->stop_queue_credit[i] = 0;
__netif_tx_unlock_bh(tx_queue);
}
return count;
}
#endif
/** pfe_eth_show_txavail
*
*/
static ssize_t pfe_eth_show_txavail(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
ssize_t len = 0;
int i;
for (i = 0; i < EMAC_TXQ_CNT; i++) {
struct netdev_queue *tx_queue = netdev_get_tx_queue(priv->dev, i);
__netif_tx_lock_bh(tx_queue);
len += sprintf(buf + len, "%d", hif_lib_tx_avail(&priv->client, i));
__netif_tx_unlock_bh(tx_queue);
if (i == (EMAC_TXQ_CNT - 1))
len += sprintf(buf + len, "\n");
else
len += sprintf(buf + len, " ");
}
return len;
}
/** pfe_eth_show_default_priority
*
*/
static ssize_t pfe_eth_show_default_priority(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
unsigned long flags;
int rc;
spin_lock_irqsave(&priv->lock, flags);
rc = sprintf(buf, "%d\n", priv->default_priority);
spin_unlock_irqrestore(&priv->lock, flags);
return rc;
}
/** pfe_eth_set_default_priority
*
*/
static ssize_t pfe_eth_set_default_priority(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct pfe_eth_priv_s *priv = netdev_priv(to_net_dev(dev));
unsigned long flags;
spin_lock_irqsave(&priv->lock, flags);
priv->default_priority = simple_strtoul(buf, NULL, 0);
spin_unlock_irqrestore(&priv->lock, flags);
return count;
}
#if defined(CONFIG_SMP) && (NR_CPUS > 1)
static DEVICE_ATTR(rx_cpu_affinity, 0644, pfe_eth_show_rx_cpu_affinity, pfe_eth_set_rx_cpu_affinity);
#endif
static DEVICE_ATTR(txavail, 0444, pfe_eth_show_txavail, NULL);
static DEVICE_ATTR(default_priority, 0644, pfe_eth_show_default_priority, pfe_eth_set_default_priority);
#ifdef PFE_ETH_NAPI_STATS
static DEVICE_ATTR(napi_stats, 0644, pfe_eth_show_napi_stats, pfe_eth_set_napi_stats);
#endif
#ifdef PFE_ETH_TX_STATS
static DEVICE_ATTR(tx_stats, 0644, pfe_eth_show_tx_stats, pfe_eth_set_tx_stats);
#endif
#ifdef PFE_ETH_TSO_STATS
static DEVICE_ATTR(tso_stats, 0644, pfe_eth_show_tso_stats, pfe_eth_set_tso_stats);
#endif
#ifdef PFE_ETH_LRO_STATS
static DEVICE_ATTR(lro_nb_stats, 0644, pfe_eth_show_lro_nb_stats, pfe_eth_set_lro_nb_stats);
static DEVICE_ATTR(lro_len_stats, 0644, pfe_eth_show_lro_len_stats, pfe_eth_set_lro_len_stats);
#endif
/** pfe_eth_sysfs_init
*
*/
static int pfe_eth_sysfs_init(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
int err;
/* Initialize the default values */
/* By default, packets without conntrack will use this default high priority queue */
priv->default_priority = 15;
/* Create our sysfs files */
err = device_create_file(&dev->dev, &dev_attr_default_priority);
if (err) {
netdev_err(dev, "failed to create default_priority sysfs files\n");
goto err_priority;
}
err = device_create_file(&dev->dev, &dev_attr_txavail);
if (err) {
netdev_err(dev, "failed to create default_priority sysfs files\n");
goto err_txavail;
}
#ifdef PFE_ETH_NAPI_STATS
err = device_create_file(&dev->dev, &dev_attr_napi_stats);
if (err) {
netdev_err(dev, "failed to create napi stats sysfs files\n");
goto err_napi;
}
#endif
#ifdef PFE_ETH_TX_STATS
err = device_create_file(&dev->dev, &dev_attr_tx_stats);
if (err) {
netdev_err(dev, "failed to create tx stats sysfs files\n");
goto err_tx;
}
#endif
#ifdef PFE_ETH_TSO_STATS
err = device_create_file(&dev->dev, &dev_attr_tso_stats);
if (err) {
netdev_err(dev, "failed to create tso stats sysfs files\n");
goto err_tso;
}
#endif
#ifdef PFE_ETH_LRO_STATS
err = device_create_file(&dev->dev, &dev_attr_lro_nb_stats);
if (err) {
netdev_err(dev, "failed to create lro nb stats sysfs files\n");
goto err_lro_nb;
}
err = device_create_file(&dev->dev, &dev_attr_lro_len_stats);
if (err) {
netdev_err(dev, "failed to create lro len stats sysfs files\n");
goto err_lro_len;
}
#endif
#if defined(CONFIG_SMP) && (NR_CPUS > 1)
err = device_create_file(&dev->dev, &dev_attr_rx_cpu_affinity);
if (err) {
netdev_err(dev, "failed to create rx cpu affinity sysfs file\n");
goto err_rx_affinity;
}
#endif
return 0;
#if defined(CONFIG_SMP) && (NR_CPUS > 1)
err_rx_affinity:
#ifdef PFE_ETH_LRO_STATS
device_remove_file(&dev->dev, &dev_attr_lro_len_stats);
#endif
#endif
#ifdef PFE_ETH_LRO_STATS
err_lro_len:
device_remove_file(&dev->dev, &dev_attr_lro_nb_stats);
err_lro_nb:
#endif
#ifdef PFE_ETH_TSO_STATS
device_remove_file(&dev->dev, &dev_attr_tso_stats);
err_tso:
#endif
#ifdef PFE_ETH_TX_STATS
device_remove_file(&dev->dev, &dev_attr_tx_stats);
err_tx:
#endif
#ifdef PFE_ETH_NAPI_STATS
device_remove_file(&dev->dev, &dev_attr_napi_stats);
err_napi:
#endif
device_remove_file(&dev->dev, &dev_attr_txavail);
err_txavail:
device_remove_file(&dev->dev, &dev_attr_default_priority);
err_priority:
return -1;
}
/** pfe_eth_sysfs_exit
*
*/
void pfe_eth_sysfs_exit(struct net_device *dev)
{
#if defined(CONFIG_SMP) && (NR_CPUS > 1)
device_remove_file(&dev->dev, &dev_attr_rx_cpu_affinity);
#endif
#ifdef PFE_ETH_LRO_STATS
device_remove_file(&dev->dev, &dev_attr_lro_nb_stats);
device_remove_file(&dev->dev, &dev_attr_lro_len_stats);
#endif
#ifdef PFE_ETH_TSO_STATS
device_remove_file(&dev->dev, &dev_attr_tso_stats);
#endif
#ifdef PFE_ETH_TX_STATS
device_remove_file(&dev->dev, &dev_attr_tx_stats);
#endif
#ifdef PFE_ETH_NAPI_STATS
device_remove_file(&dev->dev, &dev_attr_napi_stats);
#endif
device_remove_file(&dev->dev, &dev_attr_txavail);
device_remove_file(&dev->dev, &dev_attr_default_priority);
}
/*************************************************************************/
/* ETHTOOL INTERCAE */
/*************************************************************************/
static char stat_gstrings[][ETH_GSTRING_LEN] = {
"tx- octets",
"tx- packets",
"tx- broadcast",
"tx- multicast",
"tx- pause",
"tx- 64 bytes packets",
"tx- 64 - 127 bytes packets",
"tx- 128 - 255 bytes packets",
"tx- 256 - 511 bytes packets",
"tx- 512 - 1023 bytes packets",
"tx- 1024 - 1518 bytes packets",
"tx- > 1518 bytes packets",
"tx- underruns - errors",
"tx- single collision",
"tx- multi collision",
"tx- exces. collision - errors",
"tx- late collision - errors",
"tx- deferred",
"tx- carrier sense - errors",
"rx- octets",
"rx- packets",
"rx- broadcast",
"rx- multicast",
"rx- pause",
"rx- 64 bytes packets",
"rx- 64 - 127 bytes packets",
"rx- 128 - 255 bytes packets",
"rx- 256 - 511 bytes packets",
"rx- 512 - 1023 bytes packets",
"rx- 1024 - 1518 bytes packets",
"rx- > 1518 bytes packets",
"rx- undersize -errors",
"rx- oversize - errors ",
"rx- jabbers - errors",
"rx- fcs - errors",
"rx- length - errors",
"rx- symbol - errors",
"rx- align - errors",
"rx- ressource - errors",
"rx- overrun - errors",
"rx- IP cksum - errors",
"rx- TCP cksum - errors",
"rx- UDP cksum - errors"
};
/**
* pfe_eth_gstrings - Fill in a buffer with the strings which correspond to
* the stats.
*
*/
static void pfe_eth_gstrings(struct net_device *dev, u32 stringset, u8 * buf)
{
switch (stringset) {
case ETH_SS_STATS:
memcpy(buf, stat_gstrings, (EMAC_RMON_LEN - 2) * ETH_GSTRING_LEN);
break;
default:
WARN_ON(1);
break;
}
}
/**
* pfe_eth_fill_stats - Fill in an array of 64-bit statistics from
* various sources. This array will be appended
* to the end of the ethtool_stats* structure, and
* returned to user space
*/
static void pfe_eth_fill_stats(struct net_device *dev, struct ethtool_stats *dummy, u64 * buf)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
int i;
for (i=0;i<EMAC_RMON_LEN;i++, buf++) {
*buf = readl(priv->EMAC_baseaddr + EMAC_RMON_BASE_OFST + (i << 2));
if ( ( i == EMAC_RMON_TXBYTES_POS ) || ( i == EMAC_RMON_RXBYTES_POS ) ){
i++;
*buf |= (u64)readl(priv->EMAC_baseaddr + EMAC_RMON_BASE_OFST + (i << 2)) << 32;
}
}
}
/**
* pfe_eth_stats_count - Returns the number of stats (and their corresponding strings)
*
*/
static int pfe_eth_stats_count(struct net_device *dev, int sset)
{
switch (sset) {
case ETH_SS_STATS:
return EMAC_RMON_LEN - 2;
default:
return -EOPNOTSUPP;
}
}
/**
* pfe_eth_get_drvinfo - Fills in the drvinfo structure with some basic info
*
*/
static void pfe_eth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo)
{
strncpy(drvinfo->driver, DRV_NAME, COMCERTO_INFOSTR_LEN);
strncpy(drvinfo->version, comcerto_eth_driver_version, COMCERTO_INFOSTR_LEN);
strncpy(drvinfo->fw_version, "N/A", COMCERTO_INFOSTR_LEN);
strncpy(drvinfo->bus_info, "N/A", COMCERTO_INFOSTR_LEN);
drvinfo->testinfo_len = 0;
drvinfo->regdump_len = 0;
drvinfo->eedump_len = 0;
}
/**
* pfe_eth_set_settings - Used to send commands to PHY.
*
*/
static int pfe_eth_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
struct phy_device *phydev = priv->phydev;
if (NULL == phydev)
return -ENODEV;
return phy_ethtool_sset(phydev, cmd);
}
/**
* pfe_eth_getsettings - Return the current settings in the ethtool_cmd structure.
*
*/
static int pfe_eth_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
struct phy_device *phydev = priv->phydev;
if (NULL == phydev)
return -ENODEV;
return phy_ethtool_gset(phydev, cmd);
}
/**
* pfe_eth_gemac_reglen - Return the length of the register structure.
*
*/
static int pfe_eth_gemac_reglen(struct net_device *dev)
{
return (sizeof (gemac_regs)/ sizeof(u32)) + (( MAX_UC_SPEC_ADDR_REG - 3 ) * 2);
}
/**
* pfe_eth_gemac_get_regs - Return the gemac register structure.
*
*/
static void pfe_eth_gemac_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *regbuf)
{
int i,j;
struct pfe_eth_priv_s *priv = netdev_priv(dev);
u32 *buf = (u32 *) regbuf;
for (i = 0; i < sizeof (gemac_regs) / sizeof (u32); i++)
buf[i] = readl( priv->EMAC_baseaddr + gemac_regs[i] );
for (j = 0; j < (( MAX_UC_SPEC_ADDR_REG - 3 ) * 2); j++,i++)
buf[i] = readl( priv->EMAC_baseaddr + EMAC_SPEC5_ADD_BOT + (j<<2) );
}
/**
* pfe_eth_get_msglevel - Gets the debug message mask.
*
*/
static uint32_t pfe_eth_get_msglevel(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
return priv->msg_enable;
}
/**
* pfe_eth_set_msglevel - Sets the debug message mask.
*
*/
static void pfe_eth_set_msglevel(struct net_device *dev, uint32_t data)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
priv->msg_enable = data;
}
#define HIF_RX_COAL_MAX_CLKS (~(1<<31))
#define HIF_RX_COAL_CLKS_PER_USEC (pfe->ctrl.sys_clk/1000)
#define HIF_RX_COAL_MAX_USECS (HIF_RX_COAL_MAX_CLKS/HIF_RX_COAL_CLKS_PER_USEC)
/**
* pfe_eth_set_coalesce - Sets rx interrupt coalescing timer.
*
*/
static int pfe_eth_set_coalesce(struct net_device *dev,
struct ethtool_coalesce *ec)
{
if (ec->rx_coalesce_usecs > HIF_RX_COAL_MAX_USECS)
return -EINVAL;
if (!ec->rx_coalesce_usecs) {
writel(0, HIF_INT_COAL);
return 0;
}
writel((ec->rx_coalesce_usecs * HIF_RX_COAL_CLKS_PER_USEC) | HIF_INT_COAL_ENABLE, HIF_INT_COAL);
return 0;
}
/**
* pfe_eth_get_coalesce - Gets rx interrupt coalescing timer value.
*
*/
static int pfe_eth_get_coalesce(struct net_device *dev,
struct ethtool_coalesce *ec)
{
int reg_val = readl(HIF_INT_COAL);
if (reg_val & HIF_INT_COAL_ENABLE)
ec->rx_coalesce_usecs = (reg_val & HIF_RX_COAL_MAX_CLKS) / HIF_RX_COAL_CLKS_PER_USEC;
else
ec->rx_coalesce_usecs = 0;
return 0;
}
/**
* pfe_eth_pause_rx_enabled - Tests if pause rx is enabled on GEM
*
*/
static int pfe_eth_pause_rx_enabled(struct pfe_eth_priv_s *priv)
{
return (readl(priv->EMAC_baseaddr + EMAC_NETWORK_CONFIG) & EMAC_ENABLE_PAUSE_RX) != 0;
}
/**
* pfe_eth_set_pauseparam - Sets pause parameters
*
*/
static int pfe_eth_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
if (epause->rx_pause)
{
gemac_enable_pause_rx(priv->EMAC_baseaddr);
if (priv->phydev)
priv->phydev->advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
}
else
{
gemac_disable_pause_rx(priv->EMAC_baseaddr);
if (priv->phydev)
priv->phydev->advertising &= ~(ADVERTISED_Pause | ADVERTISED_Asym_Pause);
}
return 0;
}
/**
* pfe_eth_get_pauseparam - Gets pause parameters
*
*/
static void pfe_eth_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
epause->autoneg = 0;
epause->tx_pause = 0;
epause->rx_pause = pfe_eth_pause_rx_enabled(priv);
}
struct ethtool_ops pfe_ethtool_ops = {
.get_settings = pfe_eth_get_settings,
.set_settings = pfe_eth_set_settings,
.get_drvinfo = pfe_eth_get_drvinfo,
.get_regs_len = pfe_eth_gemac_reglen,
.get_regs = pfe_eth_gemac_get_regs,
.get_link = ethtool_op_get_link,
.get_strings = pfe_eth_gstrings,
.get_sset_count = pfe_eth_stats_count,
.get_ethtool_stats = pfe_eth_fill_stats,
.get_msglevel = pfe_eth_get_msglevel,
.set_msglevel = pfe_eth_set_msglevel,
.set_coalesce = pfe_eth_set_coalesce,
.get_coalesce = pfe_eth_get_coalesce,
.set_pauseparam = pfe_eth_set_pauseparam,
.get_pauseparam = pfe_eth_get_pauseparam,
};
/** pfe_eth_mdio_reset
*/
static int pfe_eth_mdio_reset(struct mii_bus *bus)
{
struct pfe_eth_priv_s *priv = (struct pfe_eth_priv_s *)bus->priv;
netif_info(priv, drv, priv->dev, "%s\n", __func__);
#if !defined(CONFIG_PLATFORM_EMULATION)
mutex_lock(&bus->mdio_lock);
/* Setup the MII Mgmt clock speed */
gemac_set_mdc_div(priv->EMAC_baseaddr, priv->mdc_div);
/* Reset the management interface */
__raw_writel(__raw_readl(priv->EMAC_baseaddr + EMAC_NETWORK_CONTROL) | EMAC_MDIO_EN,
priv->EMAC_baseaddr + EMAC_NETWORK_CONTROL);
/* Wait until the bus is free */
while(!(__raw_readl(priv->EMAC_baseaddr + EMAC_NETWORK_STATUS) & EMAC_PHY_IDLE));
mutex_unlock(&bus->mdio_lock);
#endif
return 0;
}
/** pfe_eth_gemac_phy_timeout
*
*/
static int pfe_eth_gemac_phy_timeout(struct pfe_eth_priv_s *priv, int timeout)
{
while(!(__raw_readl(priv->EMAC_baseaddr + EMAC_NETWORK_STATUS) & EMAC_PHY_IDLE)) {
if (timeout-- <= 0) {
return -1;
}
udelay(10);
}
return 0;
}
/** pfe_eth_mdio_write
*/
static int pfe_eth_mdio_write(struct mii_bus *bus, int mii_id, int regnum, u16 value)
{
struct pfe_eth_priv_s *priv = (struct pfe_eth_priv_s *)bus->priv;
u32 write_data;
#if !defined(CONFIG_PLATFORM_EMULATION)
netif_info(priv, hw, priv->dev, "%s: phy %d\n", __func__, mii_id);
// netif_info(priv, hw, priv->dev, "%s %d %d %x\n", bus->id, mii_id, regnum, value);
write_data = 0x50020000;
write_data |= ((mii_id << 23) | (regnum << 18) | value);
__raw_writel(write_data, priv->EMAC_baseaddr + EMAC_PHY_MANAGEMENT);
if (pfe_eth_gemac_phy_timeout(priv, EMAC_MDIO_TIMEOUT)){
netdev_err(priv->dev, "%s: phy MDIO write timeout\n", __func__);
return -1;
}
#endif
return 0;
}
/** pfe_eth_mdio_read
*/
static int pfe_eth_mdio_read(struct mii_bus *bus, int mii_id, int regnum)
{
struct pfe_eth_priv_s *priv = (struct pfe_eth_priv_s *)bus->priv;
u16 value = 0;
u32 write_data;
#if !defined(CONFIG_PLATFORM_EMULATION)
netif_info(priv, hw, priv->dev, "%s: phy %d\n", __func__, mii_id);
write_data = 0x60020000;
write_data |= ((mii_id << 23) | (regnum << 18));
__raw_writel(write_data, priv->EMAC_baseaddr + EMAC_PHY_MANAGEMENT);
if (pfe_eth_gemac_phy_timeout( priv, EMAC_MDIO_TIMEOUT)) {
netdev_err(priv->dev, "%s: phy MDIO read timeout\n", __func__);
return -1;
}
value = __raw_readl(priv->EMAC_baseaddr + EMAC_PHY_MANAGEMENT) & 0xFFFF;
#endif
// netif_info(priv, hw, priv->dev, "%s %d %d %x\n", bus->id, mii_id, regnum, value);
return value;
}
/** pfe_eth_mdio_init
*/
static int pfe_eth_mdio_init(struct pfe_eth_priv_s *priv, struct comcerto_mdio_platform_data *minfo)
{
struct mii_bus *bus;
int rc;
netif_info(priv, drv, priv->dev, "%s\n", __func__);
#if !defined(CONFIG_PLATFORM_EMULATION)
bus = mdiobus_alloc();
if (!bus) {
netdev_err(priv->dev, "mdiobus_alloc() failed\n");
rc = -ENOMEM;
goto err0;
}
bus->name = "Comcerto MDIO Bus";
bus->read = &pfe_eth_mdio_read;
bus->write = &pfe_eth_mdio_write;
bus->reset = &pfe_eth_mdio_reset;
snprintf(bus->id, MII_BUS_ID_SIZE, "comcerto-%x", priv->id);
bus->priv = priv;
bus->phy_mask = minfo->phy_mask;
priv->mdc_div = minfo->mdc_div;
if (!priv->mdc_div)
priv->mdc_div = 64;
bus->irq = minfo->irq;
bus->parent = priv->pfe->dev;
netif_info(priv, drv, priv->dev, "%s: mdc_div: %d, phy_mask: %x \n", __func__, priv->mdc_div, bus->phy_mask);
rc = mdiobus_register(bus);
if (rc) {
netdev_err(priv->dev, "mdiobus_register(%s) failed\n", bus->name);
goto err1;
}
priv->mii_bus = bus;
return 0;
err1:
mdiobus_free(bus);
err0:
return rc;
#else
return 0;
#endif
}
/** pfe_eth_mdio_exit
*/
static void pfe_eth_mdio_exit(struct mii_bus *bus)
{
if (!bus)
return;
netif_info((struct pfe_eth_priv_s *)bus->priv, drv, ((struct pfe_eth_priv_s *)(bus->priv))->dev, "%s\n", __func__);
mdiobus_unregister(bus);
mdiobus_free(bus);
}
/** pfe_get_interface
*/
static phy_interface_t pfe_get_interface(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
u32 mii_mode = priv->einfo->mii_config;
netif_info(priv, drv, dev, "%s\n", __func__);
if (priv->einfo->gemac_mode & (GEMAC_SW_CONF)) {
switch (mii_mode) {
case CONFIG_COMCERTO_USE_GMII:
return PHY_INTERFACE_MODE_GMII;
break;
case CONFIG_COMCERTO_USE_RGMII:
return PHY_INTERFACE_MODE_RGMII;
break;
case CONFIG_COMCERTO_USE_RMII:
return PHY_INTERFACE_MODE_RMII;
break;
case CONFIG_COMCERTO_USE_SGMII:
return PHY_INTERFACE_MODE_SGMII;
break;
default :
case CONFIG_COMCERTO_USE_MII:
return PHY_INTERFACE_MODE_MII;
break;
}
} else {
// Bootstrap config read from controller
BUG();
return 0;
}
}
/** pfe_get_phydev_speed
*/
static int pfe_get_phydev_speed(struct phy_device *phydev)
{
switch (phydev->speed) {
case 10:
return SPEED_10M;
case 100:
return SPEED_100M;
case 1000:
default:
return SPEED_1000M;
}
}
/** pfe_get_phydev_duplex
*/
static int pfe_get_phydev_duplex(struct phy_device *phydev)
{
return ( phydev->duplex == DUPLEX_HALF ) ? DUP_HALF:DUP_FULL ;
}
/** pfe_eth_adjust_link
*/
static void pfe_eth_adjust_link(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
unsigned long flags;
struct phy_device *phydev = priv->phydev;
int new_state = 0;
netif_info(priv, drv, dev, "%s\n", __func__);
spin_lock_irqsave(&priv->lock, flags);
if (phydev->link) {
/* Now we make sure that we can be in full duplex mode.
* If not, we operate in half-duplex mode. */
if (phydev->duplex != priv->oldduplex) {
new_state = 1;
gemac_set_duplex(priv->EMAC_baseaddr, pfe_get_phydev_duplex(phydev));
priv->oldduplex = phydev->duplex;
}
if (phydev->speed != priv->oldspeed) {
new_state = 1;
gemac_set_speed(priv->EMAC_baseaddr, pfe_get_phydev_speed(phydev));
priv->oldspeed = phydev->speed;
}
if (!priv->oldlink) {
new_state = 1;
priv->oldlink = 1;
}
} else if (priv->oldlink) {
new_state = 1;
priv->oldlink = 0;
priv->oldspeed = 0;
priv->oldduplex = -1;
}
if (new_state && netif_msg_link(priv))
phy_print_status(phydev);
spin_unlock_irqrestore(&priv->lock, flags);
}
/** pfe_phy_exit
*/
static void pfe_phy_exit(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
netif_info(priv, drv, dev, "%s\n", __func__);
phy_disconnect(priv->phydev);
priv->phydev = NULL;
}
/** pfe_eth_stop
*/
static void pfe_eth_stop( struct net_device *dev )
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
netif_info(priv, drv, dev, "%s\n", __func__);
gemac_disable(priv->EMAC_baseaddr);
gpi_disable(priv->GPI_baseaddr);
if (priv->phydev)
phy_stop(priv->phydev);
}
/** pfe_eth_start
*/
static int pfe_eth_start( struct pfe_eth_priv_s *priv )
{
netif_info(priv, drv, priv->dev, "%s\n", __func__);
if (priv->phydev)
phy_start(priv->phydev);
gpi_enable(priv->GPI_baseaddr);
gemac_enable(priv->EMAC_baseaddr);
return 0;
}
/** pfe_phy_init
*
*/
static int pfe_phy_init(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
struct phy_device *phydev;
char phy_id[MII_BUS_ID_SIZE + 3];
char bus_id[MII_BUS_ID_SIZE];
phy_interface_t interface;
priv->oldlink = 0;
priv->oldspeed = 0;
priv->oldduplex = -1;
sprintf(bus_id, "comcerto-%d", priv->einfo->bus_id);
snprintf(phy_id, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, priv->einfo->phy_id);
netif_info(priv, drv, dev, "%s: %s\n", __func__, phy_id);
interface = pfe_get_interface(dev);
priv->oldlink = 0;
priv->oldspeed = 0;
priv->oldduplex = -1;
phydev = phy_connect(dev, phy_id, &pfe_eth_adjust_link, 0, interface);
if (IS_ERR(phydev)) {
netdev_err(dev, "phy_connect() failed\n");
return PTR_ERR(phydev);
}
priv->phydev = phydev;
phydev->supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
if (pfe_eth_pause_rx_enabled(priv))
phydev->advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
else
phydev->advertising &= ~(ADVERTISED_Pause | ADVERTISED_Asym_Pause);
return 0;
}
/** pfe_gemac_init
*/
static int pfe_gemac_init(struct pfe_eth_priv_s *priv)
{
GEMAC_CFG cfg;
netif_info(priv, ifup, priv->dev, "%s\n", __func__);
/* software config */
/* MII interface mode selection */
switch (priv->einfo->mii_config) {
case CONFIG_COMCERTO_USE_GMII:
cfg.mode = GMII;
break;
case CONFIG_COMCERTO_USE_MII:
cfg.mode = MII;
break;
case CONFIG_COMCERTO_USE_RGMII:
cfg.mode = RGMII;
break;
case CONFIG_COMCERTO_USE_RMII:
cfg.mode = RMII;
break;
case CONFIG_COMCERTO_USE_SGMII:
cfg.mode = SGMII;
break;
default:
cfg.mode = RGMII;
}
/* Speed selection */
switch (priv->einfo->gemac_mode & GEMAC_SW_SPEED_1G ) {
case GEMAC_SW_SPEED_1G:
cfg.speed = SPEED_1000M;
break;
case GEMAC_SW_SPEED_100M:
cfg.speed = SPEED_100M;
break;
case GEMAC_SW_SPEED_10M:
cfg.speed = SPEED_10M;
break;
default:
cfg.speed = SPEED_1000M;
}
/* Duplex selection */
cfg.duplex = ( priv->einfo->gemac_mode & GEMAC_SW_FULL_DUPLEX ) ? DUPLEX_FULL : DUPLEX_HALF;
gemac_set_config( priv->EMAC_baseaddr, &cfg);
gemac_allow_broadcast( priv->EMAC_baseaddr );
gemac_disable_unicast( priv->EMAC_baseaddr );
gemac_disable_multicast( priv->EMAC_baseaddr );
gemac_disable_fcs_rx( priv->EMAC_baseaddr );
gemac_enable_1536_rx( priv->EMAC_baseaddr );
gemac_enable_rx_jmb( priv->EMAC_baseaddr );
gemac_enable_pause_rx( priv->EMAC_baseaddr );
gemac_set_bus_width(priv->EMAC_baseaddr, 64);
/*GEM will perform checksum verifications*/
if (priv->dev->features & NETIF_F_RXCSUM)
gemac_enable_rx_checksum_offload(priv->EMAC_baseaddr);
else
gemac_disable_rx_checksum_offload(priv->EMAC_baseaddr);
return 0;
}
/** pfe_eth_event_handler
*/
static int pfe_eth_event_handler(void *data, int event, int qno)
{
struct pfe_eth_priv_s *priv = data;
switch (event) {
case EVENT_RX_PKT_IND:
if (qno == 0) {
if (napi_schedule_prep(&priv->high_napi)) {
netif_info(priv, intr, priv->dev, "%s: schedule high prio poll\n", __func__);
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_SCHED_COUNT]++;
#endif
__napi_schedule(&priv->high_napi);
}
}
else if (qno == 1) {
if (napi_schedule_prep(&priv->low_napi)) {
netif_info(priv, intr, priv->dev, "%s: schedule low prio poll\n", __func__);
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_SCHED_COUNT]++;
#endif
__napi_schedule(&priv->low_napi);
}
}
else if (qno == 2) {
if (napi_schedule_prep(&priv->lro_napi)) {
netif_info(priv, intr, priv->dev, "%s: schedule lro prio poll\n", __func__);
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_SCHED_COUNT]++;
#endif
__napi_schedule(&priv->lro_napi);
}
}
break;
case EVENT_TXDONE_IND:
case EVENT_HIGH_RX_WM:
default:
break;
}
return 0;
}
/** pfe_eth_open
*/
static int pfe_eth_open( struct net_device *dev )
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
struct hif_client_s *client;
int rc;
netif_info(priv, ifup, dev, "%s\n", __func__);
/* Register client driver with HIF */
client = &priv->client;
memset(client, 0, sizeof(*client));
client->id = PFE_CL_GEM0 + priv->id;
client->tx_qn = EMAC_TXQ_CNT;
client->rx_qn = EMAC_RXQ_CNT;
client->priv = priv;
client->pfe = priv->pfe;
client->event_handler = pfe_eth_event_handler;
client->user_cpu_id = priv->cpu_id;
/* FIXME : For now hif lib sets all tx and rx queues to same size */
client->tx_qsize = EMAC_TXQ_DEPTH;
client->rx_qsize = EMAC_RXQ_DEPTH;
if ((rc = hif_lib_client_register(client))) {
netdev_err(dev, "%s: hif_lib_client_register(%d) failed\n", __func__, client->id);
goto err0;
}
netif_info(priv, drv, dev, "%s: registered client: %p\n", __func__, client);
/* Enable gemac tx clock */
clk_enable(priv->gemtx_clk);
pfe_gemac_init(priv);
if (!is_valid_ether_addr(dev->dev_addr)) {
netdev_err(dev, "%s: invalid MAC address\n", __func__);
rc = -EADDRNOTAVAIL;
goto err1;
}
gemac_set_laddrN( priv->EMAC_baseaddr, ( MAC_ADDR *)dev->dev_addr, 1 );
if (!(priv->einfo->phy_flags & GEMAC_NO_PHY)) {
rc = pfe_phy_init(dev);
if (rc) {
netdev_err(dev, "%s: pfe_phy_init() failed\n", __func__);
goto err2;
}
}
napi_enable(&priv->high_napi);
napi_enable(&priv->low_napi);
napi_enable(&priv->lro_napi);
rc = pfe_eth_start( priv );
netif_tx_wake_all_queues(dev);
pfe_ctrl_set_eth_state(priv->id, 1, dev->dev_addr);
priv->tx_timer.expires = jiffies + ( COMCERTO_TX_RECOVERY_TIMEOUT_MS * HZ )/1000;
add_timer(&priv->tx_timer);
return 0;
err2:
pfe_ctrl_set_eth_state(priv->id, 0, NULL);
err1:
hif_lib_client_unregister(&priv->client);
clk_disable(priv->gemtx_clk);
err0:
return rc;
}
#define TX_POLL_TIMEOUT_MS 1000
/** pfe_eth_close
*/
static int pfe_eth_close( struct net_device *dev )
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
int i, qstatus;
unsigned long next_poll = jiffies + 1, end = jiffies + (TX_POLL_TIMEOUT_MS * HZ) / 1000;
int tx_pkts, prv_tx_pkts;
netif_info(priv, ifdown, dev, "%s\n", __func__);
del_timer_sync(&priv->tx_timer);
for(i = 0; i < EMAC_TXQ_CNT; i++)
hrtimer_cancel(&priv->fast_tx_timeout[i].timer);
netif_tx_stop_all_queues(dev);
pfe_eth_flush_tx(priv, 1);
/*Disable transmit in PFE before disabling GEMAC */
pfe_ctrl_set_eth_state(priv->id, 0, NULL);
prv_tx_pkts = tmu_pkts_processed(priv->id);
/*Wait till TMU transmits all pending packets
* poll tmu_qstatus and pkts processed by TMU for every 10ms
* Consider TMU is busy, If we see TMU qeueu pending or any packets processed by TMU
*/
while(1) {
if (time_after(jiffies, next_poll)) {
tx_pkts = tmu_pkts_processed(priv->id);
qstatus = tmu_qstatus(priv->id) & 0x7ffff;
if(!qstatus && (tx_pkts == prv_tx_pkts)) {
break;
}
/*Don't wait forever, break if we cross max timeout(TX_POLL_TIMEOUT_MS) */
if (time_after(jiffies, end)) {
printk(KERN_ERR "TMU%d is busy after %dmsec\n", priv->id, TX_POLL_TIMEOUT_MS);
break;
}
prv_tx_pkts = tx_pkts;
next_poll++;
}
if (need_resched())
schedule();
}
/* Wait for some more time to complete transmitting packet if any */
next_poll = jiffies + 1;
while(1) {
if (time_after(jiffies, next_poll))
break;
if (need_resched())
schedule();
}
pfe_eth_stop(dev);
napi_disable(&priv->lro_napi);
napi_disable(&priv->low_napi);
napi_disable(&priv->high_napi);
if (!(priv->einfo->phy_flags & GEMAC_NO_PHY))
pfe_phy_exit(dev);
/* Disable gemac tx clock */
clk_disable(priv->gemtx_clk);
hif_lib_client_unregister(&priv->client);
return 0;
}
/** pfe_eth_get_priority
*
*/
static int pfe_eth_get_queuenum( struct pfe_eth_priv_s *priv, struct sk_buff *skb )
{
int queuenum;
unsigned long flags;
/* Get the FPP queue number from the CONNMARK value */
#if defined(CONFIG_IP_NF_CONNTRACK_MARK) || defined(CONFIG_NF_CONNTRACK_MARK)
if (skb->nfct) {
enum ip_conntrack_info cinfo;
struct nf_conn *ct;
ct = nf_ct_get(skb, &cinfo);
if (ct) {
u_int32_t connmark;
connmark = ct->mark;
if ((connmark & 0x80000000) && priv->id != 0)
connmark >>= 16;
queuenum = connmark & EMAC_QUEUENUM_MASK;
}
else
queuenum = 0;
}
else /* continued after #endif ... */
#endif
if (skb->mark & EMAC_QUEUENUM_MASK)
queuenum = skb->mark & EMAC_QUEUENUM_MASK;
else{
spin_lock_irqsave(&priv->lock, flags);
queuenum = priv->default_priority & EMAC_QUEUENUM_MASK;
spin_unlock_irqrestore(&priv->lock, flags);
}
return queuenum;
}
/** pfe_eth_might_stop_tx
*
*/
static int pfe_eth_might_stop_tx(struct pfe_eth_priv_s *priv, int queuenum, struct netdev_queue *tx_queue, unsigned int n_desc, unsigned int n_segs)
{
int tried = 0;
ktime_t kt;
try_again:
if (unlikely((__hif_tx_avail(&pfe->hif) < n_desc)
|| (hif_lib_tx_avail(&priv->client, queuenum) < n_desc)
|| (hif_lib_tx_credit_avail(pfe, priv->id, queuenum) < n_segs))) {
if (!tried) {
hif_tx_unlock(&pfe->hif);
pfe_eth_flush_txQ(priv, queuenum, 1, n_desc);
hif_lib_update_credit(&priv->client, queuenum);
tried = 1;
hif_tx_lock(&pfe->hif);
goto try_again;
}
#ifdef PFE_ETH_TX_STATS
if (__hif_tx_avail(&pfe->hif) < n_desc)
priv->stop_queue_hif[queuenum]++;
else if (hif_lib_tx_avail(&priv->client, queuenum) < n_desc) {
priv->stop_queue_hif_client[queuenum]++;
}
else if (hif_lib_tx_credit_avail(pfe, priv->id, queuenum) < n_segs) {
priv->stop_queue_credit[queuenum]++;
}
priv->stop_queue_total[queuenum]++;
#endif
netif_tx_stop_queue(tx_queue);
kt = ktime_set(0, COMCERTO_TX_FAST_RECOVERY_TIMEOUT_MS * NSEC_PER_MSEC);
hrtimer_start(&priv->fast_tx_timeout[queuenum].timer, kt, HRTIMER_MODE_REL);
return -1;
}
else {
return 0;
}
}
#define SA_MAX_OP 2
/** pfe_hif_send_packet
*
* At this level if TX fails we drop the packet
*/
static void pfe_hif_send_packet( struct sk_buff *skb, struct pfe_eth_priv_s *priv, int queuenum)
{
struct skb_shared_info *sh = skb_shinfo(skb);
unsigned int nr_frags, nr_bytes;
u32 ctrl = 0;
#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
int i;
u16 sah[SA_MAX_OP] = {0};
struct hif_ipsec_hdr *hif_ipsec;
#endif
netif_info(priv, tx_queued, priv->dev, "%s\n", __func__);
if (skb_is_gso(skb)) {
if(likely(nr_bytes = pfe_tso(skb, &priv->client, &priv->tso, queuenum, 0))) {
hif_lib_tx_credit_use(pfe, priv->id, queuenum, sh->gso_segs);
priv->stats.tx_packets += sh->gso_segs;
priv->stats.tx_bytes += nr_bytes;
}
else
priv->stats.tx_dropped++;
return;
}
if (skb->ip_summed == CHECKSUM_PARTIAL) {
if (skb->len > 1522) {
*(u16*)(skb_transport_header(skb) + skb->csum_offset) = csum_fold(csum_partial(skb_transport_header(skb), skb->len - skb_transport_offset(skb), 0));
skb->ip_summed = 0;
ctrl = 0;
}
else
ctrl = HIF_CTRL_TX_CHECKSUM;
}
#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
/* check if packet sent from Host to PFE needs IPsec processing */
if (skb->ipsec_offload)
{
if (skb->sp)
{
for (i = skb->sp->len-1; i >= 0; i--)
{
struct xfrm_state *x = skb->sp->xvec[i];
sah[i] = htons(x->handle);
}
ctrl |= HIF_CTRL_TX_IPSEC_OUT;
/* add SA info to the hif header*/
hif_ipsec = (struct hif_ipsec_hdr *)(skb->data - sizeof(struct hif_ipsec_hdr));
hif_ipsec->sa_handle[0] = sah[0];
hif_ipsec->sa_handle[1] = sah[1];
skb->data -= sizeof(struct hif_ipsec_hdr);
skb->len += sizeof(struct hif_ipsec_hdr);
}
else
printk(KERN_ERR "%s: secure path data not found\n", __func__);
}
#endif
nr_frags = sh->nr_frags;
if (nr_frags) {
skb_frag_t *f;
int i;
__hif_lib_xmit_pkt(&priv->client, queuenum, skb->data, skb_headlen(skb), ctrl, HIF_FIRST_BUFFER, skb);
for (i = 0; i < nr_frags - 1; i++) {
f = &sh->frags[i];
__hif_lib_xmit_pkt(&priv->client, queuenum, skb_frag_address(f), skb_frag_size(f), 0x0, 0x0, skb);
}
f = &sh->frags[i];
__hif_lib_xmit_pkt(&priv->client, queuenum, skb_frag_address(f), skb_frag_size(f), 0x0, HIF_LAST_BUFFER|HIF_DATA_VALID, skb);
netif_info(priv, tx_queued, priv->dev, "%s: pkt sent successfully skb:%p nr_frags:%d len:%d\n", __func__, skb, nr_frags, skb->len);
}
else {
__hif_lib_xmit_pkt(&priv->client, queuenum, skb->data, skb->len, ctrl, HIF_FIRST_BUFFER | HIF_LAST_BUFFER | HIF_DATA_VALID, skb);
netif_info(priv, tx_queued, priv->dev, "%s: pkt sent successfully skb:%p len:%d\n", __func__, skb, skb->len);
}
hif_tx_dma_start();
priv->stats.tx_packets++;
priv->stats.tx_bytes += skb->len;
hif_lib_tx_credit_use(pfe, priv->id, queuenum, 1);
}
/** pfe_eth_flush_txQ
*/
static void pfe_eth_flush_txQ(struct pfe_eth_priv_s *priv, int txQ_num, int from_tx, int n_desc)
{
struct sk_buff *skb;
struct netdev_queue *tx_queue = netdev_get_tx_queue(priv->dev, txQ_num);
int count = max(TX_FREE_MAX_COUNT, n_desc);
unsigned int flags;
netif_info(priv, tx_done, priv->dev, "%s\n", __func__);
if (!from_tx)
__netif_tx_lock(tx_queue, smp_processor_id());
/* Clean HIF and client queue */
while (count && (skb = hif_lib_tx_get_next_complete(&priv->client, txQ_num, &flags, count))) {
/* FIXME : Invalid data can be skipped in hif_lib itself */
if (flags & HIF_DATA_VALID) {
#ifdef ETH_HIF_NODMA_MAP
if (flags & HIF_DONT_DMA_MAP) {
pfe_tx_skb_unmap(skb);
}
#endif
dev_kfree_skb_any(skb);
}
// When called from the timer, flush all descriptors
if (from_tx)
count--;
}
if (!from_tx)
__netif_tx_unlock(tx_queue);
}
/** pfe_eth_flush_tx
*/
static void pfe_eth_flush_tx(struct pfe_eth_priv_s *priv, int force)
{
int ii;
netif_info(priv, tx_done, priv->dev, "%s\n", __func__);
for (ii = 0; ii < EMAC_TXQ_CNT; ii++) {
if (force || (time_after(jiffies, priv->client.tx_q[ii].jiffies_last_packet + (COMCERTO_TX_RECOVERY_TIMEOUT_MS * HZ)/1000))) {
pfe_eth_flush_txQ(priv, ii, 0, 0); //We will release everything we can based on from_tx param, so the count param can be set to any value
hif_lib_update_credit(&priv->client, ii);
}
}
}
/** pfe_eth_send_packet
*/
static int pfe_eth_send_packet(struct sk_buff *skb, struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
int txQ_num = skb_get_queue_mapping(skb);
int n_desc, n_segs, count;
struct netdev_queue *tx_queue = netdev_get_tx_queue(priv->dev, txQ_num);
netif_info(priv, tx_queued, dev, "%s\n", __func__);
if ((!skb_is_gso(skb)) && (skb_headroom(skb) < (PFE_PKT_HEADER_SZ + sizeof(unsigned long)))) {
netif_warn(priv, tx_err, priv->dev, "%s: copying skb\n", __func__);
if (pskb_expand_head(skb, (PFE_PKT_HEADER_SZ + sizeof(unsigned long)), 0, GFP_ATOMIC)) {
/* No need to re-transmit, no way to recover*/
kfree_skb(skb);
priv->stats.tx_dropped++;
return NETDEV_TX_OK;
}
}
pfe_tx_get_req_desc(skb, &n_desc, &n_segs);
hif_tx_lock(&pfe->hif);
if(unlikely(pfe_eth_might_stop_tx(priv, txQ_num, tx_queue, n_desc, n_segs))) {
#ifdef PFE_ETH_TX_STATS
if(priv->was_stopped[txQ_num]) {
priv->clean_fail[txQ_num]++;
priv->was_stopped[txQ_num] = 0;
}
#endif
hif_tx_unlock(&pfe->hif);
return NETDEV_TX_BUSY;
}
pfe_hif_send_packet(skb, priv, txQ_num);
hif_tx_unlock(&pfe->hif);
dev->trans_start = jiffies;
// Recycle buffers if a socket's send buffer becomes half full or if the HIF client queue starts filling up
if (((count = (hif_lib_tx_pending(&priv->client, txQ_num) - HIF_CL_TX_FLUSH_MARK)) > 0)
|| (skb->sk && ((sk_wmem_alloc_get(skb->sk) << 1) > skb->sk->sk_sndbuf)))
pfe_eth_flush_txQ(priv, txQ_num, 1, count);
#ifdef PFE_ETH_TX_STATS
priv->was_stopped[txQ_num] = 0;
#endif
return NETDEV_TX_OK;
}
/** pfe_eth_select_queue
*
*/
static u16 pfe_eth_select_queue( struct net_device *dev, struct sk_buff *skb )
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
return pfe_eth_get_queuenum(priv, skb);
}
/** pfe_eth_get_stats
*/
static struct net_device_stats *pfe_eth_get_stats(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
netif_info(priv, drv, dev, "%s\n", __func__);
return &priv->stats;
}
/** pfe_eth_change_mtu
*/
static int pfe_eth_change_mtu(struct net_device *dev, int new_mtu)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
int oldsize = dev->mtu ;
int frame_size = new_mtu + ETH_HLEN +4;
netif_info(priv, drv, dev, "%s\n", __func__);
if ((frame_size < 64) || (frame_size > JUMBO_FRAME_SIZE)) {
netif_err(priv, drv, dev, "Invalid MTU setting\n");
return -EINVAL;
}
if ((new_mtu > 1500) && (dev->features & NETIF_F_TSO))
{
netdev_err(dev, "MTU cannot be set to more than 1500 while TSO is enabled. TSO must be disabled first.\n");
return -EINVAL;
}
/* Only stop and start the controller if it isn't already
* stopped, and we changed something */
if ((oldsize != new_mtu) && (dev->flags & IFF_UP)){
netdev_err(dev, "Can not change MTU - fast_path must be disabled and ifconfig down must be issued first\n");
return -EINVAL;
}
dev->mtu = new_mtu;
return 0;
}
/** pfe_eth_set_mac_address
*/
static int pfe_eth_set_mac_address(struct net_device *dev, void *addr)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
struct sockaddr *sa = addr;
netif_info(priv, drv, dev, "%s\n", __func__);
if (!is_valid_ether_addr(sa->sa_data))
return -EADDRNOTAVAIL;
memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN);
gemac_set_laddrN(priv->EMAC_baseaddr, (MAC_ADDR *)dev->dev_addr, 1);
return 0;
}
/** pfe_eth_enet_addr_byte_mac
*/
int pfe_eth_enet_addr_byte_mac(u8 * enet_byte_addr, MAC_ADDR *enet_addr)
{
if ((enet_byte_addr == NULL) || (enet_addr == NULL))
{
return -1;
}
else
{
enet_addr->bottom = enet_byte_addr[0] |
(enet_byte_addr[1] << 8) |
(enet_byte_addr[2] << 16) |
(enet_byte_addr[3] << 24);
enet_addr->top = enet_byte_addr[4] |
(enet_byte_addr[5] << 8);
return 0;
}
}
/** pfe_eth_get_hash
*/
static int pfe_eth_get_hash(u8 * addr)
{
u8 temp1,temp2,temp3,temp4,temp5,temp6,temp7,temp8;
temp1 = addr[0] & 0x3F ;
temp2 = ((addr[0] & 0xC0) >> 6)| ((addr[1] & 0x0F) << 2);
temp3 = ((addr[1] & 0xF0) >> 4) | ((addr[2] & 0x03) << 4);
temp4 = (addr[2] & 0xFC) >> 2;
temp5 = addr[3] & 0x3F;
temp6 = ((addr[3] & 0xC0) >> 6) | ((addr[4] & 0x0F) << 2);
temp7 = ((addr[4] & 0xF0) >>4 ) | ((addr[5] & 0x03) << 4);
temp8 = ((addr[5] &0xFC) >> 2);
return (temp1 ^ temp2 ^ temp3 ^ temp4 ^ temp5 ^ temp6 ^ temp7 ^ temp8);
}
/** pfe_eth_set_multi
*/
static void pfe_eth_set_multi(struct net_device *dev)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
MAC_ADDR hash_addr; /* hash register structure */
MAC_ADDR spec_addr; /* specific mac address register structure */
int result; /* index into hash register to set.. */
int uc_count = 0;
struct netdev_hw_addr *ha;
if (dev->flags & IFF_PROMISC) {
netif_info(priv, drv, dev, "entering promiscuous mode\n");
priv->promisc = 1;
gemac_enable_copy_all(priv->EMAC_baseaddr);
} else {
priv->promisc = 0;
gemac_disable_copy_all(priv->EMAC_baseaddr);
}
/* Enable broadcast frame reception if required. */
if (dev->flags & IFF_BROADCAST) {
gemac_allow_broadcast(priv->EMAC_baseaddr);
} else {
netif_info(priv, drv, dev, "disabling broadcast frame reception\n");
gemac_no_broadcast(priv->EMAC_baseaddr);
}
if (dev->flags & IFF_ALLMULTI) {
/* Set the hash to rx all multicast frames */
hash_addr.bottom = 0xFFFFFFFF;
hash_addr.top = 0xFFFFFFFF;
gemac_set_hash(priv->EMAC_baseaddr, &hash_addr);
gemac_enable_multicast(priv->EMAC_baseaddr);
netdev_for_each_uc_addr(ha, dev) {
if(uc_count >= MAX_UC_SPEC_ADDR_REG) break;
pfe_eth_enet_addr_byte_mac(ha->addr, &spec_addr);
gemac_set_laddrN(priv->EMAC_baseaddr, &spec_addr, uc_count + 2);
uc_count++;
}
} else if ((netdev_mc_count(dev) > 0) || (netdev_uc_count(dev))) {
u8 *addr;
hash_addr.bottom = 0;
hash_addr.top = 0;
netdev_for_each_mc_addr(ha, dev) {
addr = ha->addr;
netif_info(priv, drv, dev, "adding multicast address %X:%X:%X:%X:%X:%X to gem filter\n",
addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
result = pfe_eth_get_hash(addr);
if (result >= EMAC_HASH_REG_BITS) {
break;
} else {
if (result < 32) {
hash_addr.bottom |= (1 << result);
} else {
hash_addr.top |= (1 << (result - 32));
}
}
}
uc_count = -1;
netdev_for_each_uc_addr(ha, dev) {
addr = ha->addr;
if(++uc_count < MAX_UC_SPEC_ADDR_REG)
{
netdev_info(dev, "adding unicast address %02x:%02x:%02x:%02x:%02x:%02x to gem filter\n",
addr[0], addr[1], addr[2],
addr[3], addr[4], addr[5]);
pfe_eth_enet_addr_byte_mac(addr, &spec_addr);
gemac_set_laddrN(priv->EMAC_baseaddr, &spec_addr, uc_count + 2);
}
else
{
netif_info(priv, drv, dev, "adding unicast address %02x:%02x:%02x:%02x:%02x:%02x to gem hash\n",
addr[0], addr[1], addr[2],
addr[3], addr[4], addr[5]);
result = pfe_eth_get_hash(addr);
if (result >= EMAC_HASH_REG_BITS) {
break;
} else {
if (result < 32)
hash_addr.bottom |= (1 << result);
else
hash_addr.top |= (1 << (result - 32));
}
}
}
gemac_set_hash(priv->EMAC_baseaddr, &hash_addr);
if(netdev_mc_count(dev))
gemac_enable_multicast(priv->EMAC_baseaddr);
else
gemac_disable_multicast(priv->EMAC_baseaddr);
}
if(netdev_uc_count(dev) >= MAX_UC_SPEC_ADDR_REG)
gemac_enable_unicast(priv->EMAC_baseaddr);
else
{
/* Check if there are any specific address HW registers that need
* to be flushed
* */
for(uc_count = netdev_uc_count(dev); uc_count < MAX_UC_SPEC_ADDR_REG; uc_count++)
gemac_clear_laddrN(priv->EMAC_baseaddr, uc_count + 2);
gemac_disable_unicast(priv->EMAC_baseaddr);
}
if (dev->flags & IFF_LOOPBACK) {
gemac_set_loop(priv->EMAC_baseaddr, LB_LOCAL);
}
return;
}
/** pfe_eth_set_features
*/
static int pfe_eth_set_features(struct net_device *dev, u32 features)
{
struct pfe_eth_priv_s *priv = netdev_priv(dev);
int rc = 0;
if (features & NETIF_F_RXCSUM)
gemac_enable_rx_checksum_offload(priv->EMAC_baseaddr);
else
gemac_disable_rx_checksum_offload(priv->EMAC_baseaddr);
if (features & NETIF_F_LRO) {
if (pfe_ctrl_set_lro(1) < 0)
rc = -1;
} else {
if (pfe_ctrl_set_lro(0) < 0)
rc = -1;
}
return rc;
}
/** pfe_eth_fix_features
*/
static u32 pfe_eth_fix_features(struct net_device *dev, u32 features)
{
if (dev->mtu > 1500)
{
if (features & (NETIF_F_TSO))
{
features &= ~(NETIF_F_TSO);
netdev_err(dev, "TSO cannot be enabled when the MTU is larger than 1500. Please set the MTU to 1500 or lower first.\n");
}
}
return features;
}
/** pfe_eth_tx_timeout
*/
void pfe_eth_tx_timeout(unsigned long data )
{
struct pfe_eth_priv_s *priv = (struct pfe_eth_priv_s *)data;
netif_info(priv, timer, priv->dev, "%s\n", __func__);
pfe_eth_flush_tx(priv, 0);
priv->tx_timer.expires = jiffies + ( COMCERTO_TX_RECOVERY_TIMEOUT_MS * HZ )/1000;
add_timer(&priv->tx_timer);
}
/** pfe_eth_fast_tx_timeout
*/
static enum hrtimer_restart pfe_eth_fast_tx_timeout(struct hrtimer *timer)
{
struct pfe_eth_fast_timer *fast_tx_timeout = container_of(timer, struct pfe_eth_fast_timer, timer);
struct pfe_eth_priv_s *priv = container_of(fast_tx_timeout->base, struct pfe_eth_priv_s, fast_tx_timeout);
struct netdev_queue *tx_queue = netdev_get_tx_queue(priv->dev, fast_tx_timeout->queuenum);
if(netif_tx_queue_stopped(tx_queue)) {
#ifdef PFE_ETH_TX_STATS
priv->was_stopped[fast_tx_timeout->queuenum] = 1;
#endif
netif_tx_wake_queue(tx_queue);
}
return HRTIMER_NORESTART;
}
/** pfe_eth_fast_tx_timeout_init
*/
static void pfe_eth_fast_tx_timeout_init(struct pfe_eth_priv_s *priv)
{
int i;
for (i = 0; i < EMAC_TXQ_CNT; i++) {
priv->fast_tx_timeout[i].queuenum = i;
hrtimer_init(&priv->fast_tx_timeout[i].timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
priv->fast_tx_timeout[i].timer.function = pfe_eth_fast_tx_timeout;
priv->fast_tx_timeout[i].base = priv->fast_tx_timeout;
}
}
static struct sk_buff *pfe_eth_rx_skb(struct net_device *dev, struct pfe_eth_priv_s *priv, unsigned int qno)
{
void *buf_addr;
unsigned int rx_ctrl;
unsigned int desc_ctrl = 0;
struct hif_ipsec_hdr *ipsec_hdr;
struct sk_buff *skb;
struct sk_buff *skb_frag, *skb_frag_last = NULL;
int length = 0, offset;
#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
struct timespec ktime;
#endif
skb = priv->skb_inflight[qno];
if (skb && (skb_frag_last = skb_shinfo(skb)->frag_list)) {
while (skb_frag_last->next)
skb_frag_last = skb_frag_last->next;
}
while (!(desc_ctrl & CL_DESC_LAST)) {
buf_addr = hif_lib_receive_pkt(&priv->client, qno, &length, &offset, &rx_ctrl, &desc_ctrl, (void **)&ipsec_hdr);
if (!buf_addr)
goto incomplete;
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_DESC_COUNT]++;
#endif
/* First frag */
if (desc_ctrl & CL_DESC_FIRST) {
#if defined(CONFIG_PLATFORM_EMULATION) || defined(CONFIG_PLATFORM_PCI)
skb = dev_alloc_skb(PFE_BUF_SIZE);
if (unlikely(!skb)) {
goto pkt_drop;
}
skb_copy_to_linear_data(skb, buf_addr, length + offset);
kfree(buf_addr);
#else
#if defined(CONFIG_COMCERTO_ZONE_DMA_NCNB)
skb = alloc_skb(length + offset + 32, GFP_ATOMIC);
#else
skb = alloc_skb_header(PFE_BUF_SIZE, buf_addr, GFP_ATOMIC);
#endif
if (unlikely(!skb)) {
goto pkt_drop;
}
#endif
skb_reserve(skb, offset);
#if defined(CONFIG_COMCERTO_ZONE_DMA_NCNB)
__memcpy(skb->data, buf_addr + offset, length);
kfree(buf_addr);
#endif
skb_put(skb, length);
skb->dev = dev;
if ((dev->features & NETIF_F_RXCSUM) && (rx_ctrl & HIF_CTRL_RX_CHECKSUMMED))
skb->ip_summed = CHECKSUM_UNNECESSARY;
else
skb_checksum_none_assert(skb);
#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
if (rx_ctrl & HIF_CTRL_RX_IPSEC_IN) {
if (ipsec_hdr) {
struct sec_path *sp;
struct xfrm_state *x;
unsigned short *sah = &ipsec_hdr->sa_handle[0];
int i = 0;
sp = secpath_dup(skb->sp);
if (!sp)
{
kfree_skb(skb);
goto pkt_drop;
}
skb->sp = sp;
/* at maximum 2 SA are expected */
while (i <= 1)
{
if(!*sah)
break;
if ((x = xfrm_state_lookup_byhandle(dev_net(dev), ntohs(*sah))) == NULL)
{
kfree_skb(skb);
goto pkt_drop;
}
sp->xvec[i] = x;
if (!x->curlft.use_time)
{
ktime = current_kernel_time();
x->curlft.use_time = (unsigned long)ktime.tv_sec;
}
i++; sah++;
}
sp->len = i;
}
}
#endif
} else {
/* Next frags */
if (unlikely(!skb)) {
printk(KERN_ERR "%s: NULL skb_inflight\n", __func__);
goto pkt_drop;
}
#if defined(CONFIG_COMCERTO_ZONE_DMA_NCNB)
skb_frag = alloc_skb(length + offset + 32, GFP_ATOMIC);
#else
skb_frag = alloc_skb_header(PFE_BUF_SIZE, buf_addr, GFP_ATOMIC);
#endif
if (unlikely(!skb_frag)) {
kfree(buf_addr);
goto pkt_drop;
}
skb_reserve(skb_frag, offset);
#if defined(CONFIG_COMCERTO_ZONE_DMA_NCNB)
__memcpy(skb_frag->data, buf_addr + offset, length);
kfree(buf_addr);
#endif
skb_put(skb_frag, length);
skb_frag->dev = dev;
if (skb_shinfo(skb)->frag_list)
skb_frag_last->next = skb_frag;
else
skb_shinfo(skb)->frag_list = skb_frag;
skb->truesize += skb_frag->truesize;
skb->data_len += length;
skb->len += length;
skb_frag_last = skb_frag;
}
}
priv->skb_inflight[qno] = NULL;
return skb;
incomplete:
priv->skb_inflight[qno] = skb;
return NULL;
pkt_drop:
priv->skb_inflight[qno] = NULL;
if (skb) {
kfree_skb(skb);
} else {
kfree(buf_addr);
}
priv->stats.rx_errors++;
return NULL;
}
static struct sk_buff *pfe_eth_rx_page(struct net_device *dev, struct pfe_eth_priv_s *priv, unsigned int qno)
{
struct page *p;
void *buf_addr;
unsigned int rx_ctrl;
unsigned int desc_ctrl;
struct sk_buff *skb;
int length, offset, data_offset;
struct hif_lro_hdr *lro_hdr;
u32 pe_id;
while (1) {
buf_addr = hif_lib_receive_pkt(&priv->client, qno, &length, &offset, &rx_ctrl, &desc_ctrl, (void **)&lro_hdr);
if (!buf_addr)
goto empty;
if (qno == 2)
pe_id = (rx_ctrl >> HIF_CTRL_RX_PE_ID_OFST) & 0xf;
else
pe_id = 0;
skb = priv->skb_inflight[qno + pe_id];
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_DESC_COUNT]++;
#endif
/* First frag */
if ((desc_ctrl & CL_DESC_FIRST) && !skb) {
p = virt_to_page(buf_addr);
skb = dev_alloc_skb(MAX_HDR_SIZE + PFE_PKT_HEADROOM + 2);
if (unlikely(!skb)) {
goto pkt_drop;
}
skb_reserve(skb, PFE_PKT_HEADROOM + 2);
if (lro_hdr) {
data_offset = lro_hdr->data_offset;
if (lro_hdr->mss)
skb_shinfo(skb)->gso_size = lro_hdr->mss;
// printk(KERN_INFO "mss: %d, offset: %d, data_offset: %d, len: %d\n", lro_hdr->mss, offset, lro_hdr->data_offset, length);
} else {
data_offset = MAX_HDR_SIZE;
}
/* We don't need the fragment if the whole packet */
/* has been copied in the first linear skb */
if (length <= data_offset) {
__memcpy(skb->data, buf_addr + offset, length);
skb_put(skb, length);
free_page((unsigned long)buf_addr);
} else {
__memcpy(skb->data, buf_addr + offset, data_offset);
skb_put(skb, data_offset);
skb_add_rx_frag(skb, 0, p, offset + data_offset, length - data_offset);
}
skb->dev = dev;
if ((dev->features & NETIF_F_RXCSUM) && (rx_ctrl & HIF_CTRL_RX_CHECKSUMMED))
skb->ip_summed = CHECKSUM_UNNECESSARY;
else
skb_checksum_none_assert(skb);
} else {
/* Next frags */
if (unlikely(!skb)) {
printk(KERN_ERR "%s: NULL skb_inflight\n", __func__);
goto pkt_drop;
}
p = virt_to_page(buf_addr);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, p, offset, length);
}
/* Last buffer in a software chain */
if ((desc_ctrl & CL_DESC_LAST) && !(rx_ctrl & HIF_CTRL_RX_CONTINUED))
break;
/* Keep track of skb for this queue/pe */
priv->skb_inflight[qno + pe_id] = skb;
}
priv->skb_inflight[qno + pe_id] = NULL;
return skb;
pkt_drop:
priv->skb_inflight[qno + pe_id] = NULL;
if (skb) {
kfree_skb(skb);
} else {
free_page((unsigned long)buf_addr);
}
priv->stats.rx_errors++;
return NULL;
empty:
return NULL;
}
/** pfe_eth_poll
*/
static int pfe_eth_poll(struct pfe_eth_priv_s *priv, struct napi_struct *napi, unsigned int qno, int budget)
{
struct net_device *dev = priv->dev;
struct sk_buff *skb;
int work_done = 0;
unsigned int len;
netif_info(priv, intr, priv->dev, "%s\n", __func__);
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_POLL_COUNT]++;
#endif
do {
if (page_mode)
skb = pfe_eth_rx_page(dev, priv, qno);
else
skb = pfe_eth_rx_skb(dev, priv, qno);
if (!skb)
break;
len = skb->len;
/* Packet will be processed */
skb->protocol = eth_type_trans(skb, dev);
#ifdef PFE_ETH_LRO_STATS
priv->lro_len_counters[((u32)skb->len >> 11) & (LRO_LEN_COUNT_MAX - 1)]++;
priv->lro_nb_counters[skb_shinfo(skb)->nr_frags & (LRO_NB_COUNT_MAX - 1)]++;
#endif
netif_receive_skb(skb);
priv->stats.rx_packets++;
priv->stats.rx_bytes += len;
dev->last_rx = jiffies;
work_done++;
#ifdef PFE_ETH_NAPI_STATS
priv->napi_counters[NAPI_PACKET_COUNT]++;
#endif
} while (work_done < budget);
/* If no Rx receive nor cleanup work was done, exit polling mode.
* No more netif_running(dev) check is required here , as this is checked in
* net/core/dev.c ( 2.6.33.5 kernel specific).
*/
if (work_done < budget) {
napi_complete(napi);
hif_lib_event_handler_start(&priv->client, EVENT_RX_PKT_IND, qno);
}
#ifdef PFE_ETH_NAPI_STATS
else
priv->napi_counters[NAPI_FULL_BUDGET_COUNT]++;
#endif
return work_done;
}
/** pfe_eth_lro_poll
*/
static int pfe_eth_lro_poll(struct napi_struct *napi, int budget)
{
struct pfe_eth_priv_s *priv = container_of(napi, struct pfe_eth_priv_s, lro_napi);
netif_info(priv, intr, priv->dev, "%s\n", __func__);
return pfe_eth_poll(priv, napi, 2, budget);
}
/** pfe_eth_low_poll
*/
static int pfe_eth_low_poll(struct napi_struct *napi, int budget)
{
struct pfe_eth_priv_s *priv = container_of(napi, struct pfe_eth_priv_s, low_napi);
netif_info(priv, intr, priv->dev, "%s\n", __func__);
return pfe_eth_poll(priv, napi, 1, budget);
}
/** pfe_eth_high_poll
*/
static int pfe_eth_high_poll(struct napi_struct *napi, int budget )
{
struct pfe_eth_priv_s *priv = container_of(napi, struct pfe_eth_priv_s, high_napi);
netif_info(priv, intr, priv->dev, "%s\n", __func__);
return pfe_eth_poll(priv, napi, 0, budget);
}
static const struct net_device_ops pfe_netdev_ops = {
.ndo_open = pfe_eth_open,
.ndo_stop = pfe_eth_close,
.ndo_start_xmit = pfe_eth_send_packet,
.ndo_select_queue = pfe_eth_select_queue,
.ndo_get_stats = pfe_eth_get_stats,
.ndo_change_mtu = pfe_eth_change_mtu,
.ndo_set_mac_address = pfe_eth_set_mac_address,
.ndo_set_rx_mode = pfe_eth_set_multi,
.ndo_set_features = pfe_eth_set_features,
.ndo_fix_features = pfe_eth_fix_features,
.ndo_validate_addr = eth_validate_addr,
};
/** pfe_eth_init_one
*/
static int pfe_eth_init_one( struct pfe *pfe, int id )
{
struct net_device *dev = NULL;
struct pfe_eth_priv_s *priv = NULL;
struct comcerto_eth_platform_data *einfo;
struct comcerto_mdio_platform_data *minfo;
struct comcerto_pfe_platform_data *pfe_info;
int err;
/* Extract pltform data */
#if defined(CONFIG_PLATFORM_EMULATION) || defined(CONFIG_PLATFORM_PCI)
pfe_info = (struct comcerto_pfe_platform_data *) &comcerto_pfe_pdata;
#else
pfe_info = (struct comcerto_pfe_platform_data *) pfe->dev->platform_data;
#endif
if (!pfe_info) {
printk(KERN_ERR "%s: pfe missing additional platform data\n", __func__);
err = -ENODEV;
goto err0;
}
einfo = (struct comcerto_eth_platform_data *) pfe_info->comcerto_eth_pdata;
/* einfo never be NULL, but no harm in having this check */
if (!einfo) {
printk(KERN_ERR "%s: pfe missing additional gemacs platform data\n", __func__);
err = -ENODEV;
goto err0;
}
minfo = (struct comcerto_mdio_platform_data *) pfe_info->comcerto_mdio_pdata;
/* einfo never be NULL, but no harm in having this check */
if (!minfo) {
printk(KERN_ERR "%s: pfe missing additional mdios platform data\n", __func__);
err = -ENODEV;
goto err0;
}
/*
* FIXME: Need to check some flag in "einfo" to know whether
* GEMAC is enabled Or not.
*/
/* Create an ethernet device instance */
dev = alloc_etherdev_mq(sizeof (*priv), EMAC_TXQ_CNT);
if (!dev) {
printk(KERN_ERR "%s: gemac %d device allocation failed\n", __func__, einfo[id].gem_id);
err = -ENOMEM;
goto err0;
}
priv = netdev_priv(dev);
priv->dev = dev;
priv->id = einfo[id].gem_id;
priv->pfe = pfe;
/* get gemac tx clock */
priv->gemtx_clk = clk_get(NULL, "gemtx");
if (IS_ERR(priv->gemtx_clk)) {
printk(KERN_ERR "%s: Unable to get the clock for gemac %d\n", __func__, priv->id);
err = -ENODEV;
goto err1;
}
pfe->eth.eth_priv[id] = priv;
/* Set the info in the priv to the current info */
priv->einfo = &einfo[id];
priv->EMAC_baseaddr = cbus_emac_base[id];
priv->GPI_baseaddr = cbus_gpi_base[id];
/* FIXME : For now TMU queue numbers hardcoded, later should be taken from pfe.h */
#define HIF_GEMAC_TMUQ_BASE 6
priv->low_tmuQ = HIF_GEMAC_TMUQ_BASE + (id * 2);
priv->high_tmuQ = priv->low_tmuQ + 1;
spin_lock_init(&priv->lock);
priv->tx_timer.data = (unsigned long)priv;
priv->tx_timer.function = pfe_eth_tx_timeout;
priv->tx_timer.expires = jiffies + ( COMCERTO_TX_RECOVERY_TIMEOUT_MS * HZ )/1000;
init_timer(&priv->tx_timer);
priv->cpu_id = -1;
pfe_eth_fast_tx_timeout_init(priv);
/* Copy the station address into the dev structure, */
memcpy(dev->dev_addr, einfo[id].mac_addr, ETH_ALEN);
err = dev_alloc_name(dev, einfo[id].name);
if (err < 0) {
netdev_err(dev, "%s: dev_alloc_name(%s) failed\n", __func__, einfo[id].name);
err = -EINVAL;
goto err2;
}
/* Initialize mdio */
if (minfo[id].enabled) {
if ((err = pfe_eth_mdio_init(priv, &minfo[id]))) {
netdev_err(dev, "%s: pfe_eth_mdio_init() failed\n", __func__);
goto err2;
}
}
dev->mtu = 1500;
/* supported features */
dev->hw_features = NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
NETIF_F_SG | NETIF_F_TSO;
/* enabled by default */
dev->features = dev->hw_features;
if (lro_mode) {
dev->hw_features |= NETIF_F_LRO;
dev->features |= NETIF_F_LRO;
pfe_ctrl_set_lro(1);
}
dev->netdev_ops = &pfe_netdev_ops;
dev->ethtool_ops = &pfe_ethtool_ops;
/* Enable basic messages by default */
priv->msg_enable = NETIF_MSG_IFUP | NETIF_MSG_IFDOWN | NETIF_MSG_LINK | NETIF_MSG_PROBE;
err = register_netdev(dev);
if (err) {
netdev_err(dev, "register_netdev() failed\n");
goto err3;
}
netif_napi_add(dev, &priv->low_napi, pfe_eth_low_poll, HIF_RX_POLL_WEIGHT - 16);
netif_napi_add(dev, &priv->high_napi, pfe_eth_high_poll, HIF_RX_POLL_WEIGHT - 16);
netif_napi_add(dev, &priv->lro_napi, pfe_eth_lro_poll, HIF_RX_POLL_WEIGHT - 16);
/* Create all the sysfs files */
if(pfe_eth_sysfs_init(dev))
goto err4;
netif_info(priv, probe, dev, "%s: created interface, baseaddr: %p\n", __func__, priv->EMAC_baseaddr);
return 0;
err4:
unregister_netdev(dev);
err3:
pfe_eth_mdio_exit(priv->mii_bus);
err2:
clk_put(priv->gemtx_clk);
err1:
free_netdev(priv->dev);
err0:
return err;
}
/** pfe_eth_init
*/
int pfe_eth_init(struct pfe *pfe)
{
int ii = 0;
int err;
printk(KERN_INFO "%s\n", __func__);
cbus_emac_base[0] = EMAC1_BASE_ADDR;
cbus_emac_base[1] = EMAC2_BASE_ADDR;
cbus_emac_base[2] = EMAC3_BASE_ADDR;
cbus_gpi_base[0] = EGPI1_BASE_ADDR;
cbus_gpi_base[1] = EGPI2_BASE_ADDR;
cbus_gpi_base[2] = EGPI3_BASE_ADDR;
for (ii = 0; ii < NUM_GEMAC_SUPPORT; ii++) {
if ((err = pfe_eth_init_one(pfe, ii)))
goto err0;
}
return 0;
err0:
while(ii--){
pfe_eth_exit_one( pfe->eth.eth_priv[ii] );
}
/* Register three network devices in the kernel */
return err;
}
/** pfe_eth_exit_one
*/
static void pfe_eth_exit_one( struct pfe_eth_priv_s *priv )
{
netif_info(priv, probe, priv->dev, "%s\n", __func__);
pfe_eth_sysfs_exit(priv->dev);
clk_put(priv->gemtx_clk);
unregister_netdev(priv->dev);
pfe_eth_mdio_exit(priv->mii_bus);
free_netdev(priv->dev);
}
/** pfe_eth_exit
*/
void pfe_eth_exit(struct pfe *pfe)
{
int ii;
printk(KERN_INFO "%s\n", __func__);
for(ii = 0; ii < NUM_GEMAC_SUPPORT; ii++ ) {
/*
* FIXME: Need to check some flag in "einfo" to know whether
* GEMAC is enabled Or not.
*/
pfe_eth_exit_one(pfe->eth.eth_priv[ii]);
}
}