cxgb4: add ETHOFLD hardware queue support

Add support for configuring and managing ETHOFLD hardware queues.
Keep the queue count and MSI-X allocation scheme same as NIC queues.
ETHOFLD hardware queues are dynamically allocated/destroyed as
TC-MQPRIO Qdisc offload is enabled/disabled on the corresponding
interface, respectively.

Signed-off-by: Rahul Lakkireddy <[email protected]>
Signed-off-by: David S. Miller <[email protected]>

drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h

@@ -325,6 +325,9 @@ enum cudbg_qdesc_qtype {
  CUDBG_QTYPE_CRYPTO_FLQ,
  CUDBG_QTYPE_TLS_RXQ,
  CUDBG_QTYPE_TLS_FLQ,
+ CUDBG_QTYPE_ETHOFLD_TXQ,
+ CUDBG_QTYPE_ETHOFLD_RXQ,
+ CUDBG_QTYPE_ETHOFLD_FLQ,
  CUDBG_QTYPE_MAX,
 };
 

drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c

@@ -2930,6 +2930,10 @@ void cudbg_fill_qdesc_num_and_size(const struct adapter *padap,
  tot_size += CXGB4_ULD_MAX * MAX_ULD_QSETS * SGE_MAX_IQ_SIZE *
  MAX_RXQ_DESC_SIZE;
 
+ /* ETHOFLD TXQ, RXQ, and FLQ */
+ tot_entries += MAX_OFLD_QSETS * 3;
+ tot_size += MAX_OFLD_QSETS * MAX_TXQ_ENTRIES * MAX_TXQ_DESC_SIZE;
+
  tot_size += sizeof(struct cudbg_ver_hdr) +
  sizeof(struct cudbg_qdesc_info) +
  sizeof(struct cudbg_qdesc_entry) * tot_entries;
@@ -3087,6 +3091,23 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init,
  }
  }
 
+ /* ETHOFLD TXQ */
+ if (s->eohw_txq)
+ for (i = 0; i < s->eoqsets; i++)
+ QDESC_GET_TXQ(&s->eohw_txq[i].q,
+ CUDBG_QTYPE_ETHOFLD_TXQ, out);
+
+ /* ETHOFLD RXQ and FLQ */
+ if (s->eohw_rxq) {
+ for (i = 0; i < s->eoqsets; i++)
+ QDESC_GET_RXQ(&s->eohw_rxq[i].rspq,
+ CUDBG_QTYPE_ETHOFLD_RXQ, out);
+
+ for (i = 0; i < s->eoqsets; i++)
+ QDESC_GET_FLQ(&s->eohw_rxq[i].fl,
+ CUDBG_QTYPE_ETHOFLD_FLQ, out);
+ }
+
 out_unlock:
  mutex_unlock(&uld_mutex);
 

drivers/net/ethernet/chelsio/cxgb4/cxgb4.h

@@ -835,6 +835,16 @@ struct sge_eosw_txq {
  struct tasklet_struct qresume_tsk; /* Restarts the queue */
 };
 
+struct sge_eohw_txq {
+ spinlock_t lock; /* Per queue lock */
+ struct sge_txq q; /* HW Txq */
+ struct adapter *adap; /* Backpointer to adapter */
+ unsigned long tso; /* # of TSO requests */
+ unsigned long tx_cso; /* # of Tx checksum offloads */
+ unsigned long vlan_ins; /* # of Tx VLAN insertions */
+ unsigned long mapping_err; /* # of I/O MMU packet mapping errors */
+};
+
 struct sge {
  struct sge_eth_txq ethtxq[MAX_ETH_QSETS];
  struct sge_eth_txq ptptxq;
@@ -848,11 +858,16 @@ struct sge {
  struct sge_rspq intrq ____cacheline_aligned_in_smp;
  spinlock_t intrq_lock;
 
+ struct sge_eohw_txq *eohw_txq;
+ struct sge_ofld_rxq *eohw_rxq;
+
  u16 max_ethqsets; /* # of available Ethernet queue sets */
  u16 ethqsets; /* # of active Ethernet queue sets */
  u16 ethtxq_rover; /* Tx queue to clean up next */
  u16 ofldqsets; /* # of active ofld queue sets */
  u16 nqs_per_uld; /* # of Rx queues per ULD */
+ u16 eoqsets; /* # of ETHOFLD queues */
+
  u16 timer_val[SGE_NTIMERS];
  u8 counter_val[SGE_NCOUNTERS];
  u16 dbqtimer_tick;
@@ -1466,6 +1481,9 @@ int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
 int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq,
  struct net_device *dev, unsigned int iqid,
  unsigned int uld_type);
+int t4_sge_alloc_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq,
+ struct net_device *dev, u32 iqid);
+void t4_sge_free_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq);
 irqreturn_t t4_sge_intr_msix(int irq, void *cookie);
 int t4_sge_init(struct adapter *adap);
 void t4_sge_start(struct adapter *adap);
@@ -1995,4 +2013,6 @@ int cxgb4_get_msix_idx_from_bmap(struct adapter *adap);
 void cxgb4_free_msix_idx_in_bmap(struct adapter *adap, u32 msix_idx);
 int cxgb_open(struct net_device *dev);
 int cxgb_close(struct net_device *dev);
+void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q);
+void cxgb4_quiesce_rx(struct sge_rspq *q);
 #endif /* __CXGB4_H__ */

drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c

@@ -2658,18 +2658,20 @@ static int sge_qinfo_uld_ciq_entries(const struct adapter *adap, int uld)
 
 static int sge_qinfo_show(struct seq_file *seq, void *v)
 {
+ int eth_entries, ctrl_entries, eo_entries = 0;
  int uld_rxq_entries[CXGB4_ULD_MAX] = { 0 };
  int uld_ciq_entries[CXGB4_ULD_MAX] = { 0 };
  int uld_txq_entries[CXGB4_TX_MAX] = { 0 };
  const struct sge_uld_txq_info *utxq_info;
  const struct sge_uld_rxq_info *urxq_info;
  struct adapter *adap = seq->private;
  int i, n, r = (uintptr_t)v - 1;
- int eth_entries, ctrl_entries;
  struct sge *s = &adap->sge;
 
  eth_entries = DIV_ROUND_UP(adap->sge.ethqsets, 4);
  ctrl_entries = DIV_ROUND_UP(MAX_CTRL_QUEUES, 4);
+ if (adap->sge.eohw_txq)
+ eo_entries = DIV_ROUND_UP(adap->sge.eoqsets, 4);
 
  mutex_lock(&uld_mutex);
  if (s->uld_txq_info)
@@ -2761,6 +2763,54 @@ do { \
  }
 
  r -= eth_entries;
+ if (r < eo_entries) {
+ int base_qset = r * 4;
+ const struct sge_ofld_rxq *rx = &s->eohw_rxq[base_qset];
+ const struct sge_eohw_txq *tx = &s->eohw_txq[base_qset];
+
+ n = min(4, s->eoqsets - 4 * r);
+
+ S("QType:", "ETHOFLD");
+ S("Interface:",
+ rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A");
+ T("TxQ ID:", q.cntxt_id);
+ T("TxQ size:", q.size);
+ T("TxQ inuse:", q.in_use);
+ T("TxQ CIDX:", q.cidx);
+ T("TxQ PIDX:", q.pidx);
+ R("RspQ ID:", rspq.abs_id);
+ R("RspQ size:", rspq.size);
+ R("RspQE size:", rspq.iqe_len);
+ R("RspQ CIDX:", rspq.cidx);
+ R("RspQ Gen:", rspq.gen);
+ S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
+ S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]);
+ R("FL ID:", fl.cntxt_id);
+ S3("u", "FL size:", rx->fl.size ? rx->fl.size - 8 : 0);
+ R("FL pend:", fl.pend_cred);
+ R("FL avail:", fl.avail);
+ R("FL PIDX:", fl.pidx);
+ R("FL CIDX:", fl.cidx);
+ RL("RxPackets:", stats.pkts);
+ RL("RxImm:", stats.imm);
+ RL("RxAN", stats.an);
+ RL("RxNoMem", stats.nomem);
+ TL("TSO:", tso);
+ TL("TxCSO:", tx_cso);
+ TL("VLANins:", vlan_ins);
+ TL("TxQFull:", q.stops);
+ TL("TxQRestarts:", q.restarts);
+ TL("TxMapErr:", mapping_err);
+ RL("FLAllocErr:", fl.alloc_failed);
+ RL("FLLrgAlcErr:", fl.large_alloc_failed);
+ RL("FLMapErr:", fl.mapping_err);
+ RL("FLLow:", fl.low);
+ RL("FLStarving:", fl.starving);
+
+ goto unlock;
+ }
+
+ r -= eo_entries;
  if (r < uld_txq_entries[CXGB4_TX_OFLD]) {
  const struct sge_uld_txq *tx;
 
@@ -3007,6 +3057,7 @@ static int sge_queue_entries(const struct adapter *adap)
  mutex_unlock(&uld_mutex);
 
  return DIV_ROUND_UP(adap->sge.ethqsets, 4) +
+ (adap->sge.eohw_txq ? DIV_ROUND_UP(adap->sge.eoqsets, 4) : 0) +
  tot_uld_entries +
  DIV_ROUND_UP(MAX_CTRL_QUEUES, 4) + 1;
 }

drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

@@ -880,6 +880,12 @@ static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
  return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 }
 
+void cxgb4_quiesce_rx(struct sge_rspq *q)
+{
+ if (q->handler)
+ napi_disable(&q->napi);
+}
+
 /*
  * Wait until all NAPI handlers are descheduled.
  */
@@ -890,8 +896,10 @@ static void quiesce_rx(struct adapter *adap)
  for (i = 0; i < adap->sge.ingr_sz; i++) {
  struct sge_rspq *q = adap->sge.ingr_map[i];
 
- if (q && q->handler)
- napi_disable(&q->napi);
+ if (!q)
+ continue;
+
+ cxgb4_quiesce_rx(q);
  }
 }
 
@@ -913,6 +921,17 @@ static void disable_interrupts(struct adapter *adap)
  }
 }
 
+void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q)
+{
+ if (q->handler)
+ napi_enable(&q->napi);
+
+ /* 0-increment GTS to start the timer and enable interrupts */
+ t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
+ SEINTARM_V(q->intr_params) |
+ INGRESSQID_V(q->cntxt_id));
+}
+
 /*
  * Enable NAPI scheduling and interrupt generation for all Rx queues.
  */
@@ -925,13 +944,8 @@ static void enable_rx(struct adapter *adap)
 
  if (!q)
  continue;
- if (q->handler)
- napi_enable(&q->napi);
 
- /* 0-increment GTS to start the timer and enable interrupts */
- t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
- SEINTARM_V(q->intr_params) |
- INGRESSQID_V(q->cntxt_id));
+ cxgb4_enable_rx(adap, q);
  }
 }
 
@@ -5360,6 +5374,19 @@ static int cfg_queues(struct adapter *adap)
  avail_qsets -= num_ulds * s->ofldqsets;
  }
 
+ /* ETHOFLD Queues used for QoS offload should follow same
+ * allocation scheme as normal Ethernet Queues.
+ */
+ if (is_ethofld(adap)) {
+ if (avail_qsets < s->max_ethqsets) {
+ adap->params.ethofld = 0;
+ s->eoqsets = 0;
+ } else {
+ s->eoqsets = s->max_ethqsets;
+ }
+ avail_qsets -= s->eoqsets;
+ }
+
  for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
  struct sge_eth_rxq *r = &s->ethrxq[i];
 
@@ -5473,9 +5500,9 @@ void cxgb4_free_msix_idx_in_bmap(struct adapter *adap,
 
 static int enable_msix(struct adapter *adap)
 {
+ u32 eth_need, uld_need = 0, ethofld_need = 0;
+ u32 ethqsets = 0, ofldqsets = 0, eoqsets = 0;
  u8 num_uld = 0, nchan = adap->params.nports;
- u32 ethqsets = 0, ofldqsets = 0;
- u32 eth_need, uld_need = 0;
  u32 i, want, need, num_vec;
  struct sge *s = &adap->sge;
  struct msix_entry *entries;
@@ -5499,6 +5526,12 @@ static int enable_msix(struct adapter *adap)
  need += uld_need;
  }
 
+ if (is_ethofld(adap)) {
+ want += s->eoqsets;
+ ethofld_need = eth_need;
+ need += ethofld_need;
+ }
+
  want += EXTRA_VECS;
  need += EXTRA_VECS;
 
@@ -5531,7 +5564,9 @@ static int enable_msix(struct adapter *adap)
  adap->params.crypto = 0;
  adap->params.ethofld = 0;
  s->ofldqsets = 0;
+ s->eoqsets = 0;
  uld_need = 0;
+ ethofld_need = 0;
  }
 
  num_vec = allocated;
@@ -5543,10 +5578,12 @@ static int enable_msix(struct adapter *adap)
  ethqsets = eth_need;
  if (is_uld(adap))
  ofldqsets = nchan;
+ if (is_ethofld(adap))
+ eoqsets = ethofld_need;
 
  num_vec -= need;
  while (num_vec) {
- if (num_vec < eth_need ||
+ if (num_vec < eth_need + ethofld_need ||
  ethqsets > s->max_ethqsets)
  break;
 
@@ -5557,6 +5594,10 @@ static int enable_msix(struct adapter *adap)
 
  ethqsets++;
  num_vec--;
+ if (ethofld_need) {
+ eoqsets++;
+ num_vec--;
+ }
  }
  }
 
@@ -5574,6 +5615,8 @@ static int enable_msix(struct adapter *adap)
  ethqsets = s->max_ethqsets;
  if (is_uld(adap))
  ofldqsets = s->ofldqsets;
+ if (is_ethofld(adap))
+ eoqsets = s->eoqsets;
  }
 
  if (ethqsets < s->max_ethqsets) {
@@ -5586,6 +5629,9 @@ static int enable_msix(struct adapter *adap)
  s->nqs_per_uld = s->ofldqsets;
  }
 
+ if (is_ethofld(adap))
+ s->eoqsets = eoqsets;
+
  /* map for msix */
  ret = alloc_msix_info(adap, allocated);
  if (ret)
@@ -5597,8 +5643,8 @@ static int enable_msix(struct adapter *adap)
  }
 
  dev_info(adap->pdev_dev,
- "%d MSI-X vectors allocated, nic %d per uld %d\n",
- allocated, s->max_ethqsets, s->nqs_per_uld);
+ "%d MSI-X vectors allocated, nic %d eoqsets %d per uld %d\n",
+ allocated, s->max_ethqsets, s->eoqsets, s->nqs_per_uld);
 
  kfree(entries);
  return 0;