sk(4): jumbo mbufs and rxring accounting

classic Classic list List threaded Threaded
19 messages Options
Reply | Threaded
Open this post in threaded view
|

sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
this is an update of if_sk.c r1.151, which tried to introduce
mclgeti. it updates it to use the if_rxring accounting.

does anyone have one they can test this on?

it also saves about 2k on amd64.

Index: if_sk.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_sk.c,v
retrieving revision 1.168
diff -u -p -r1.168 if_sk.c
--- if_sk.c 19 Apr 2014 18:29:39 -0000 1.168
+++ if_sk.c 10 Jul 2014 05:27:38 -0000
@@ -157,12 +157,10 @@ void sk_watchdog(struct ifnet *);
 int sk_ifmedia_upd(struct ifnet *);
 void sk_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 void skc_reset(struct sk_softc *);
-int sk_newbuf(struct sk_if_softc *, int, struct mbuf *, bus_dmamap_t);
-int sk_alloc_jumbo_mem(struct sk_if_softc *);
-void *sk_jalloc(struct sk_if_softc *);
-void sk_jfree(caddr_t, u_int, void *);
+int sk_newbuf(struct sk_if_softc *);
 int sk_reset(struct sk_if_softc *);
 int sk_init_rx_ring(struct sk_if_softc *);
+void sk_fill_rx_ring(struct sk_if_softc *);
 int sk_init_tx_ring(struct sk_if_softc *);
 
 int sk_xmac_miibus_readreg(struct device *, int, int);
@@ -551,21 +549,29 @@ sk_init_rx_ring(struct sk_if_softc *sc_i
  rd->sk_rx_ring[i].sk_next = htole32(SK_RX_RING_ADDR(sc_if, nexti));
  }
 
- for (i = 0; i < SK_RX_RING_CNT; i++) {
- if (sk_newbuf(sc_if, i, NULL,
-    sc_if->sk_cdata.sk_rx_jumbo_map) == ENOBUFS) {
- printf("%s: failed alloc of %dth mbuf\n",
-    sc_if->sk_dev.dv_xname, i);
- return (ENOBUFS);
- }
- }
-
  sc_if->sk_cdata.sk_rx_prod = 0;
  sc_if->sk_cdata.sk_rx_cons = 0;
 
+ if_rxr_init(&sc_if->sk_cdata.sk_rx_ring, 2, SK_RX_RING_CNT);
+
+ sk_fill_rx_ring(sc_if);
+
  return (0);
 }
 
+void
+sk_fill_rx_ring(struct sk_if_softc *sc_if)
+{
+ struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
+ u_int slots;
+
+ for (slots = if_rxr_get(rxr, SK_RX_RING_CNT); slots > 0; slots--) {
+ if (sk_newbuf(sc_if) == ENOBUFS)
+ break;
+ }
+ if_rxr_put(rxr, slots);
+}
+
 int
 sk_init_tx_ring(struct sk_if_softc *sc_if)
 {
@@ -613,199 +619,44 @@ sk_init_tx_ring(struct sk_if_softc *sc_i
 }
 
 int
-sk_newbuf(struct sk_if_softc *sc_if, int i, struct mbuf *m,
-  bus_dmamap_t dmamap)
+sk_newbuf(struct sk_if_softc *sc_if)
 {
- struct mbuf *m_new = NULL;
+ struct mbuf *m;
  struct sk_chain *c;
  struct sk_rx_desc *r;
+ bus_dmamap_t dmamap;
+ int error;
 
- if (m == NULL) {
- caddr_t buf = NULL;
-
- MGETHDR(m_new, M_DONTWAIT, MT_DATA);
- if (m_new == NULL)
- return (ENOBUFS);
-
- /* Allocate the jumbo buffer */
- buf = sk_jalloc(sc_if);
- if (buf == NULL) {
- m_freem(m_new);
- DPRINTFN(1, ("%s jumbo allocation failed -- packet "
-    "dropped!\n", sc_if->arpcom.ac_if.if_xname));
- return (ENOBUFS);
- }
-
- /* Attach the buffer to the mbuf */
- m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
- MEXTADD(m_new, buf, SK_JLEN, 0, sk_jfree, sc_if);
- } else {
- /*
- * We're re-using a previously allocated mbuf;
- * be sure to re-init pointers and lengths to
- * default values.
- */
- m_new = m;
- m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
- m_new->m_data = m_new->m_ext.ext_buf;
- }
- m_adj(m_new, ETHER_ALIGN);
-
- c = &sc_if->sk_cdata.sk_rx_chain[i];
- r = c->sk_desc;
- c->sk_mbuf = m_new;
- r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr +
-    (((vaddr_t)m_new->m_data
-             - (vaddr_t)sc_if->sk_cdata.sk_jumbo_buf)));
- r->sk_ctl = htole32(SK_JLEN | SK_RXSTAT);
 
- SK_CDRXSYNC(sc_if, i, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
+ m = MCLGETI(NULL, M_DONTWAIT, NULL, SK_JLEN);
+ if (m == NULL)
+ return (ENOBUFS);
 
- return (0);
-}
+ m_adj(m, ETHER_ALIGN);
 
-/*
- * Memory management for jumbo frames.
- */
+ dmamap = sc_if->sk_cdata.sk_rx_map[sc_if->sk_cdata.sk_rx_prod];
 
-int
-sk_alloc_jumbo_mem(struct sk_if_softc *sc_if)
-{
- struct sk_softc *sc = sc_if->sk_softc;
- caddr_t ptr, kva;
- bus_dma_segment_t seg;
- int i, rseg, state, error;
- struct sk_jpool_entry   *entry;
-
- state = error = 0;
-
- /* Grab a big chunk o' storage. */
- if (bus_dmamem_alloc(sc->sc_dmatag, SK_JMEM, PAGE_SIZE, 0,
-     &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
- printf(": can't alloc rx buffers");
+ error = bus_dmamap_load_mbuf(sc_if->sk_softc->sc_dmatag, dmamap, m,
+    BUS_DMA_READ|BUS_DMA_NOWAIT);
+ if (error) {
+ m_freem(m);
  return (ENOBUFS);
  }
 
- state = 1;
- if (bus_dmamem_map(sc->sc_dmatag, &seg, rseg, SK_JMEM, &kva,
-   BUS_DMA_NOWAIT)) {
- printf(": can't map dma buffers (%d bytes)", SK_JMEM);
- error = ENOBUFS;
- goto out;
- }
-
- state = 2;
- if (bus_dmamap_create(sc->sc_dmatag, SK_JMEM, 1, SK_JMEM, 0,
-    BUS_DMA_NOWAIT, &sc_if->sk_cdata.sk_rx_jumbo_map)) {
- printf(": can't create dma map");
- error = ENOBUFS;
- goto out;
- }
-
- state = 3;
- if (bus_dmamap_load(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_jumbo_map,
-    kva, SK_JMEM, NULL, BUS_DMA_NOWAIT)) {
- printf(": can't load dma map");
- error = ENOBUFS;
- goto out;
- }
-
- state = 4;
- sc_if->sk_cdata.sk_jumbo_buf = (caddr_t)kva;
- DPRINTFN(1,("sk_jumbo_buf = 0x%08X\n", sc_if->sk_cdata.sk_jumbo_buf));
+ bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
+    dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
 
- LIST_INIT(&sc_if->sk_jfree_listhead);
- LIST_INIT(&sc_if->sk_jinuse_listhead);
-
- /*
- * Now divide it up into 9K pieces and save the addresses
- * in an array.
- */
- ptr = sc_if->sk_cdata.sk_jumbo_buf;
- for (i = 0; i < SK_JSLOTS; i++) {
- sc_if->sk_cdata.sk_jslots[i] = ptr;
- ptr += SK_JLEN;
- entry = malloc(sizeof(struct sk_jpool_entry),
-    M_DEVBUF, M_NOWAIT);
- if (entry == NULL) {
- sc_if->sk_cdata.sk_jumbo_buf = NULL;
- printf(": no memory for jumbo buffer queue!");
- error = ENOBUFS;
- goto out;
- }
- entry->slot = i;
- LIST_INSERT_HEAD(&sc_if->sk_jfree_listhead,
- entry, jpool_entries);
- }
-out:
- if (error != 0) {
- switch (state) {
- case 4:
- bus_dmamap_unload(sc->sc_dmatag,
-    sc_if->sk_cdata.sk_rx_jumbo_map);
- case 3:
- bus_dmamap_destroy(sc->sc_dmatag,
-    sc_if->sk_cdata.sk_rx_jumbo_map);
- case 2:
- bus_dmamem_unmap(sc->sc_dmatag, kva, SK_JMEM);
- case 1:
- bus_dmamem_free(sc->sc_dmatag, &seg, rseg);
- break;
- default:
- break;
- }
- }
-
- return (error);
-}
+ c = &sc_if->sk_cdata.sk_rx_chain[sc_if->sk_cdata.sk_rx_prod];
+ c->sk_mbuf = m;
 
-/*
- * Allocate a jumbo buffer.
- */
-void *
-sk_jalloc(struct sk_if_softc *sc_if)
-{
- struct sk_jpool_entry   *entry;
-
- entry = LIST_FIRST(&sc_if->sk_jfree_listhead);
-
- if (entry == NULL)
- return (NULL);
-
- LIST_REMOVE(entry, jpool_entries);
- LIST_INSERT_HEAD(&sc_if->sk_jinuse_listhead, entry, jpool_entries);
- return (sc_if->sk_cdata.sk_jslots[entry->slot]);
-}
-
-/*
- * Release a jumbo buffer.
- */
-void
-sk_jfree(caddr_t buf, u_int size, void *arg)
-{
- struct sk_jpool_entry *entry;
- struct sk_if_softc *sc;
- int i;
-
- /* Extract the softc struct pointer. */
- sc = (struct sk_if_softc *)arg;
+ r = c->sk_desc;
+ r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr);
+ r->sk_data_hi = htole32(((u_int64_t)dmamap->dm_segs[0].ds_addr) >> 32);
+ r->sk_ctl = htole32(dmamap->dm_segs[0].ds_len | SK_RXSTAT);
 
- if (sc == NULL)
- panic("sk_jfree: can't find softc pointer!");
+ SK_CDRXSYNC(sc_if, 1, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
 
- /* calculate the slot this buffer belongs to */
- i = ((vaddr_t)buf
-     - (vaddr_t)sc->sk_cdata.sk_jumbo_buf) / SK_JLEN;
-
- if ((i < 0) || (i >= SK_JSLOTS))
- panic("sk_jfree: asked to free buffer that we don't manage!");
-
- entry = LIST_FIRST(&sc->sk_jinuse_listhead);
- if (entry == NULL)
- panic("sk_jfree: buffer not in use!");
- entry->slot = i;
- LIST_REMOVE(entry, jpool_entries);
- LIST_INSERT_HEAD(&sc->sk_jfree_listhead, entry, jpool_entries);
+ return (0);
 }
 
 /*
@@ -1001,7 +852,7 @@ sk_attach(struct device *parent, struct
  struct skc_attach_args *sa = aux;
  struct ifnet *ifp;
  caddr_t kva;
- int i;
+ int i, error;
 
  sc_if->sk_port = sa->skc_port;
  sc_if->sk_softc = sc;
@@ -1128,10 +979,14 @@ sk_attach(struct device *parent, struct
  }
         sc_if->sk_rdata = (struct sk_ring_data *)kva;
 
- /* Try to allocate memory for jumbo buffers. */
- if (sk_alloc_jumbo_mem(sc_if)) {
- printf(": jumbo buffer allocation failed\n");
- goto fail_3;
+ for (i = 0; i < SK_RX_RING_CNT; i++) {
+ bus_dmamap_create(sc->sc_dmatag, SK_JLEN, 1,
+    SK_JLEN, 0, 0, &sc_if->sk_cdata.sk_rx_map[i]);
+ if (error != 0) {
+ printf("\n%s: unable to create rx DMA map %d, "
+    "error = %d\n", sc->sk_dev.dv_xname, i, error);
+ goto fail_4;
+ }
  }
 
  ifp = &sc_if->arpcom.ac_if;
@@ -1197,12 +1052,18 @@ sk_attach(struct device *parent, struct
 
  DPRINTFN(2, ("sk_attach: end\n"));
  return;
+fail_4:
+ for (i = 0; i < SK_RX_RING_CNT; i++) {
+ if (sc_if->sk_cdata.sk_rx_map[i] == NULL)
+ continue;
 
-fail_2:
+ bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_map[i]);
+ }
+fail_3:
  bus_dmamem_unmap(sc->sc_dmatag, kva, sizeof(struct sk_ring_data));
-fail_1:
+fail_2:
  bus_dmamem_free(sc->sc_dmatag, &sc_if->sk_ring_seg, sc_if->sk_ring_nseg);
-fail_3:
+fail_1:
  bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_ring_map);
 fail:
  sc->sk_if[sa->skc_port] = NULL;
@@ -1722,45 +1583,44 @@ sk_rxeof(struct sk_if_softc *sc_if)
 {
  struct sk_softc *sc = sc_if->sk_softc;
  struct ifnet *ifp = &sc_if->arpcom.ac_if;
+ struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
  struct mbuf *m;
  struct sk_chain *cur_rx;
  struct sk_rx_desc *cur_desc;
- int i, cur, total_len = 0;
+ int cur, total_len = 0;
  u_int32_t rxstat, sk_ctl;
  bus_dmamap_t dmamap;
 
  DPRINTFN(2, ("sk_rxeof\n"));
 
- i = sc_if->sk_cdata.sk_rx_prod;
-
- for (;;) {
- cur = i;
+ while (if_rxr_inuse(rxr) > 0) {
+ cur = sc_if->sk_cdata.sk_rx_cons;
 
  /* Sync the descriptor */
  SK_CDRXSYNC(sc_if, cur,
     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 
- sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[i].sk_ctl);
- if ((sk_ctl & SK_RXCTL_OWN) != 0) {
- /* Invalidate the descriptor -- it's not ready yet */
- SK_CDRXSYNC(sc_if, cur, BUS_DMASYNC_PREREAD);
- sc_if->sk_cdata.sk_rx_prod = i;
+ cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
+ if (cur_rx->sk_mbuf == NULL)
+ break;
+
+ sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[cur].sk_ctl);
+ if ((sk_ctl & SK_RXCTL_OWN) != 0)
  break;
- }
 
- cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
  cur_desc = &sc_if->sk_rdata->sk_rx_ring[cur];
- dmamap = sc_if->sk_cdata.sk_rx_jumbo_map;
+ dmamap = sc_if->sk_cdata.sk_rx_map[cur];
 
  bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
     dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
 
- rxstat = letoh32(cur_desc->sk_xmac_rxstat);
  m = cur_rx->sk_mbuf;
  cur_rx->sk_mbuf = NULL;
- total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
+ if_rxr_put(rxr, 1);
 
- SK_INC(i, SK_RX_RING_CNT);
+ total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
+ rxstat = letoh32(cur_desc->sk_xmac_rxstat);
 
  if ((sk_ctl & (SK_RXCTL_STATUS_VALID | SK_RXCTL_FIRSTFRAG |
     SK_RXCTL_LASTFRAG)) != (SK_RXCTL_STATUS_VALID |
@@ -1769,31 +1629,12 @@ sk_rxeof(struct sk_if_softc *sc_if)
     total_len > SK_JUMBO_FRAMELEN ||
     sk_rxvalid(sc, rxstat, total_len) == 0) {
  ifp->if_ierrors++;
- sk_newbuf(sc_if, cur, m, dmamap);
+ m_freem(m);
  continue;
  }
 
- /*
- * Try to allocate a new jumbo buffer. If that
- * fails, copy the packet to mbufs and put the
- * jumbo buffer back in the ring so it can be
- * re-used. If allocating mbufs fails, then we
- * have to drop the packet.
- */
- if (sk_newbuf(sc_if, cur, NULL, dmamap) == ENOBUFS) {
- struct mbuf *m0;
- m0 = m_devget(mtod(m, char *), total_len, ETHER_ALIGN,
-    ifp);
- sk_newbuf(sc_if, cur, m, dmamap);
- if (m0 == NULL) {
- ifp->if_ierrors++;
- continue;
- }
- m = m0;
- } else {
- m->m_pkthdr.rcvif = ifp;
- m->m_pkthdr.len = m->m_len = total_len;
- }
+ m->m_pkthdr.rcvif = ifp;
+ m->m_pkthdr.len = m->m_len = total_len;
 
  ifp->if_ipackets++;
 
@@ -1805,6 +1646,8 @@ sk_rxeof(struct sk_if_softc *sc_if)
  /* pass it on. */
  ether_input_mbuf(ifp, m);
  }
+
+ sk_fill_rx_ring(sc_if);
 }
 
 void
@@ -2574,6 +2417,7 @@ sk_stop(struct sk_if_softc *sc_if, int s
 {
  struct sk_softc *sc = sc_if->sk_softc;
  struct ifnet *ifp = &sc_if->arpcom.ac_if;
+ bus_dmamap_t dmamap;
  struct sk_txmap_entry *dma;
  int i;
  u_int32_t val;
@@ -2664,6 +2508,10 @@ sk_stop(struct sk_if_softc *sc_if, int s
  /* Free RX and TX mbufs still in the queues. */
  for (i = 0; i < SK_RX_RING_CNT; i++) {
  if (sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf != NULL) {
+ dmamap = sc_if->sk_cdata.sk_rx_map[i];
+ bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
+    dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
  m_freem(sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf);
  sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf = NULL;
  }
Index: if_skvar.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_skvar.h,v
retrieving revision 1.8
diff -u -p -r1.8 if_skvar.h
--- if_skvar.h 7 Mar 2013 06:13:31 -0000 1.8
+++ if_skvar.h 10 Jul 2014 05:27:38 -0000
@@ -81,11 +81,6 @@
 #ifndef _DEV_PCI_IF_SKVAR_H_
 #define _DEV_PCI_IF_SKVAR_H_
 
-struct sk_jpool_entry {
- int                             slot;
- LIST_ENTRY(sk_jpool_entry) jpool_entries;
-};
-
 struct sk_chain {
  void *sk_desc;
  struct mbuf *sk_mbuf;
@@ -111,17 +106,12 @@ struct sk_chain_data {
  struct sk_chain sk_rx_chain[SK_RX_RING_CNT];
  struct sk_txmap_entry *sk_tx_map[SK_TX_RING_CNT];
  bus_dmamap_t sk_rx_map[SK_RX_RING_CNT];
- bus_dmamap_t sk_rx_jumbo_map;
  int sk_tx_prod;
  int sk_tx_cons;
  int sk_tx_cnt;
  int sk_rx_prod;
  int sk_rx_cons;
- int sk_rx_cnt;
- /* Stick the jumbo mem management stuff here too. */
- caddr_t sk_jslots[SK_JSLOTS];
- void *sk_jumbo_buf;
-
+ struct if_rxring sk_rx_ring;
 };
 
 struct sk_ring_data {
@@ -223,8 +213,6 @@ struct sk_if_softc {
  int sk_ring_nseg;
  struct sk_softc *sk_softc; /* parent controller */
  int sk_tx_bmu; /* TX BMU register */
- LIST_HEAD(__sk_jfreehead, sk_jpool_entry) sk_jfree_listhead;
- LIST_HEAD(__sk_jinusehead, sk_jpool_entry) sk_jinuse_listhead;
  SIMPLEQ_HEAD(__sk_txmaphead, sk_txmap_entry) sk_txmap_head;
 };
 

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Brad Smith-14
On 10/07/14 1:33 AM, David Gwynne wrote:
> this is an update of if_sk.c r1.151, which tried to introduce
> mclgeti. it updates it to use the if_rxring accounting.
>
> does anyone have one they can test this on?
>
> it also saves about 2k on amd64.

Doesn't work at all.

skc0 at pci0 dev 5 function 0 "Schneider & Koch SK-98xx" rev 0x13,
GEnesis (0x0): ivec 0x784
sk0 at skc0 port A: address 00:00:5a:98:b9:c0

skc0: unable to create rx DMA map 1, error = 0

--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5

On 11 Jul 2014, at 9:48, Brad Smith <[hidden email]> wrote:

> On 10/07/14 1:33 AM, David Gwynne wrote:
>> this is an update of if_sk.c r1.151, which tried to introduce
>> mclgeti. it updates it to use the if_rxring accounting.
>>
>> does anyone have one they can test this on?
>>
>> it also saves about 2k on amd64.
>
> Doesn't work at all.
>
> skc0 at pci0 dev 5 function 0 "Schneider & Koch SK-98xx" rev 0x13, GEnesis (0x0): ivec 0x784
> sk0 at skc0 port A: address 00:00:5a:98:b9:c0
>
> skc0: unable to create rx DMA map 1, error = 0

cool, lemme go hunting.

cheers,
dlg

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
how about this?


Index: if_sk.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_sk.c,v
retrieving revision 1.168
diff -u -p -r1.168 if_sk.c
--- if_sk.c 19 Apr 2014 18:29:39 -0000 1.168
+++ if_sk.c 12 Jul 2014 08:29:20 -0000
@@ -157,12 +157,10 @@ void sk_watchdog(struct ifnet *);
 int sk_ifmedia_upd(struct ifnet *);
 void sk_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 void skc_reset(struct sk_softc *);
-int sk_newbuf(struct sk_if_softc *, int, struct mbuf *, bus_dmamap_t);
-int sk_alloc_jumbo_mem(struct sk_if_softc *);
-void *sk_jalloc(struct sk_if_softc *);
-void sk_jfree(caddr_t, u_int, void *);
+int sk_newbuf(struct sk_if_softc *);
 int sk_reset(struct sk_if_softc *);
 int sk_init_rx_ring(struct sk_if_softc *);
+void sk_fill_rx_ring(struct sk_if_softc *);
 int sk_init_tx_ring(struct sk_if_softc *);
 
 int sk_xmac_miibus_readreg(struct device *, int, int);
@@ -551,21 +549,29 @@ sk_init_rx_ring(struct sk_if_softc *sc_i
  rd->sk_rx_ring[i].sk_next = htole32(SK_RX_RING_ADDR(sc_if, nexti));
  }
 
- for (i = 0; i < SK_RX_RING_CNT; i++) {
- if (sk_newbuf(sc_if, i, NULL,
-    sc_if->sk_cdata.sk_rx_jumbo_map) == ENOBUFS) {
- printf("%s: failed alloc of %dth mbuf\n",
-    sc_if->sk_dev.dv_xname, i);
- return (ENOBUFS);
- }
- }
-
  sc_if->sk_cdata.sk_rx_prod = 0;
  sc_if->sk_cdata.sk_rx_cons = 0;
 
+ if_rxr_init(&sc_if->sk_cdata.sk_rx_ring, 2, SK_RX_RING_CNT);
+
+ sk_fill_rx_ring(sc_if);
+
  return (0);
 }
 
+void
+sk_fill_rx_ring(struct sk_if_softc *sc_if)
+{
+ struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
+ u_int slots;
+
+ for (slots = if_rxr_get(rxr, SK_RX_RING_CNT); slots > 0; slots--) {
+ if (sk_newbuf(sc_if) == ENOBUFS)
+ break;
+ }
+ if_rxr_put(rxr, slots);
+}
+
 int
 sk_init_tx_ring(struct sk_if_softc *sc_if)
 {
@@ -613,199 +619,44 @@ sk_init_tx_ring(struct sk_if_softc *sc_i
 }
 
 int
-sk_newbuf(struct sk_if_softc *sc_if, int i, struct mbuf *m,
-  bus_dmamap_t dmamap)
+sk_newbuf(struct sk_if_softc *sc_if)
 {
- struct mbuf *m_new = NULL;
+ struct mbuf *m;
  struct sk_chain *c;
  struct sk_rx_desc *r;
+ bus_dmamap_t dmamap;
+ int error;
 
- if (m == NULL) {
- caddr_t buf = NULL;
-
- MGETHDR(m_new, M_DONTWAIT, MT_DATA);
- if (m_new == NULL)
- return (ENOBUFS);
-
- /* Allocate the jumbo buffer */
- buf = sk_jalloc(sc_if);
- if (buf == NULL) {
- m_freem(m_new);
- DPRINTFN(1, ("%s jumbo allocation failed -- packet "
-    "dropped!\n", sc_if->arpcom.ac_if.if_xname));
- return (ENOBUFS);
- }
-
- /* Attach the buffer to the mbuf */
- m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
- MEXTADD(m_new, buf, SK_JLEN, 0, sk_jfree, sc_if);
- } else {
- /*
- * We're re-using a previously allocated mbuf;
- * be sure to re-init pointers and lengths to
- * default values.
- */
- m_new = m;
- m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
- m_new->m_data = m_new->m_ext.ext_buf;
- }
- m_adj(m_new, ETHER_ALIGN);
-
- c = &sc_if->sk_cdata.sk_rx_chain[i];
- r = c->sk_desc;
- c->sk_mbuf = m_new;
- r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr +
-    (((vaddr_t)m_new->m_data
-             - (vaddr_t)sc_if->sk_cdata.sk_jumbo_buf)));
- r->sk_ctl = htole32(SK_JLEN | SK_RXSTAT);
 
- SK_CDRXSYNC(sc_if, i, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
+ m = MCLGETI(NULL, M_DONTWAIT, NULL, SK_JLEN);
+ if (m == NULL)
+ return (ENOBUFS);
 
- return (0);
-}
+ m_adj(m, ETHER_ALIGN);
 
-/*
- * Memory management for jumbo frames.
- */
+ dmamap = sc_if->sk_cdata.sk_rx_map[sc_if->sk_cdata.sk_rx_prod];
 
-int
-sk_alloc_jumbo_mem(struct sk_if_softc *sc_if)
-{
- struct sk_softc *sc = sc_if->sk_softc;
- caddr_t ptr, kva;
- bus_dma_segment_t seg;
- int i, rseg, state, error;
- struct sk_jpool_entry   *entry;
-
- state = error = 0;
-
- /* Grab a big chunk o' storage. */
- if (bus_dmamem_alloc(sc->sc_dmatag, SK_JMEM, PAGE_SIZE, 0,
-     &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
- printf(": can't alloc rx buffers");
+ error = bus_dmamap_load_mbuf(sc_if->sk_softc->sc_dmatag, dmamap, m,
+    BUS_DMA_READ|BUS_DMA_NOWAIT);
+ if (error) {
+ m_freem(m);
  return (ENOBUFS);
  }
 
- state = 1;
- if (bus_dmamem_map(sc->sc_dmatag, &seg, rseg, SK_JMEM, &kva,
-   BUS_DMA_NOWAIT)) {
- printf(": can't map dma buffers (%d bytes)", SK_JMEM);
- error = ENOBUFS;
- goto out;
- }
-
- state = 2;
- if (bus_dmamap_create(sc->sc_dmatag, SK_JMEM, 1, SK_JMEM, 0,
-    BUS_DMA_NOWAIT, &sc_if->sk_cdata.sk_rx_jumbo_map)) {
- printf(": can't create dma map");
- error = ENOBUFS;
- goto out;
- }
-
- state = 3;
- if (bus_dmamap_load(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_jumbo_map,
-    kva, SK_JMEM, NULL, BUS_DMA_NOWAIT)) {
- printf(": can't load dma map");
- error = ENOBUFS;
- goto out;
- }
-
- state = 4;
- sc_if->sk_cdata.sk_jumbo_buf = (caddr_t)kva;
- DPRINTFN(1,("sk_jumbo_buf = 0x%08X\n", sc_if->sk_cdata.sk_jumbo_buf));
+ bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
+    dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
 
- LIST_INIT(&sc_if->sk_jfree_listhead);
- LIST_INIT(&sc_if->sk_jinuse_listhead);
+ c = &sc_if->sk_cdata.sk_rx_chain[sc_if->sk_cdata.sk_rx_prod];
+ c->sk_mbuf = m;
 
- /*
- * Now divide it up into 9K pieces and save the addresses
- * in an array.
- */
- ptr = sc_if->sk_cdata.sk_jumbo_buf;
- for (i = 0; i < SK_JSLOTS; i++) {
- sc_if->sk_cdata.sk_jslots[i] = ptr;
- ptr += SK_JLEN;
- entry = malloc(sizeof(struct sk_jpool_entry),
-    M_DEVBUF, M_NOWAIT);
- if (entry == NULL) {
- sc_if->sk_cdata.sk_jumbo_buf = NULL;
- printf(": no memory for jumbo buffer queue!");
- error = ENOBUFS;
- goto out;
- }
- entry->slot = i;
- LIST_INSERT_HEAD(&sc_if->sk_jfree_listhead,
- entry, jpool_entries);
- }
-out:
- if (error != 0) {
- switch (state) {
- case 4:
- bus_dmamap_unload(sc->sc_dmatag,
-    sc_if->sk_cdata.sk_rx_jumbo_map);
- case 3:
- bus_dmamap_destroy(sc->sc_dmatag,
-    sc_if->sk_cdata.sk_rx_jumbo_map);
- case 2:
- bus_dmamem_unmap(sc->sc_dmatag, kva, SK_JMEM);
- case 1:
- bus_dmamem_free(sc->sc_dmatag, &seg, rseg);
- break;
- default:
- break;
- }
- }
-
- return (error);
-}
-
-/*
- * Allocate a jumbo buffer.
- */
-void *
-sk_jalloc(struct sk_if_softc *sc_if)
-{
- struct sk_jpool_entry   *entry;
-
- entry = LIST_FIRST(&sc_if->sk_jfree_listhead);
-
- if (entry == NULL)
- return (NULL);
-
- LIST_REMOVE(entry, jpool_entries);
- LIST_INSERT_HEAD(&sc_if->sk_jinuse_listhead, entry, jpool_entries);
- return (sc_if->sk_cdata.sk_jslots[entry->slot]);
-}
-
-/*
- * Release a jumbo buffer.
- */
-void
-sk_jfree(caddr_t buf, u_int size, void *arg)
-{
- struct sk_jpool_entry *entry;
- struct sk_if_softc *sc;
- int i;
-
- /* Extract the softc struct pointer. */
- sc = (struct sk_if_softc *)arg;
+ r = c->sk_desc;
+ r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr);
+ r->sk_data_hi = htole32(((u_int64_t)dmamap->dm_segs[0].ds_addr) >> 32);
+ r->sk_ctl = htole32(dmamap->dm_segs[0].ds_len | SK_RXSTAT);
 
- if (sc == NULL)
- panic("sk_jfree: can't find softc pointer!");
+ SK_CDRXSYNC(sc_if, 1, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
 
- /* calculate the slot this buffer belongs to */
- i = ((vaddr_t)buf
-     - (vaddr_t)sc->sk_cdata.sk_jumbo_buf) / SK_JLEN;
-
- if ((i < 0) || (i >= SK_JSLOTS))
- panic("sk_jfree: asked to free buffer that we don't manage!");
-
- entry = LIST_FIRST(&sc->sk_jinuse_listhead);
- if (entry == NULL)
- panic("sk_jfree: buffer not in use!");
- entry->slot = i;
- LIST_REMOVE(entry, jpool_entries);
- LIST_INSERT_HEAD(&sc->sk_jfree_listhead, entry, jpool_entries);
+ return (0);
 }
 
 /*
@@ -873,6 +724,11 @@ sk_ioctl(struct ifnet *ifp, u_long comma
  error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
  break;
 
+ case SIOCGIFRXR:
+ error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
+    NULL, SK_JLEN, &sc_if->sk_cdata.sk_rx_ring);
+ break;
+
  default:
  error = ether_ioctl(ifp, &sc_if->arpcom, command, data);
  }
@@ -1001,7 +857,7 @@ sk_attach(struct device *parent, struct
  struct skc_attach_args *sa = aux;
  struct ifnet *ifp;
  caddr_t kva;
- int i;
+ int i, error;
 
  sc_if->sk_port = sa->skc_port;
  sc_if->sk_softc = sc;
@@ -1128,10 +984,14 @@ sk_attach(struct device *parent, struct
  }
         sc_if->sk_rdata = (struct sk_ring_data *)kva;
 
- /* Try to allocate memory for jumbo buffers. */
- if (sk_alloc_jumbo_mem(sc_if)) {
- printf(": jumbo buffer allocation failed\n");
- goto fail_3;
+ for (i = 0; i < SK_RX_RING_CNT; i++) {
+ error = bus_dmamap_create(sc->sc_dmatag, SK_JLEN, 1,
+    SK_JLEN, 0, 0, &sc_if->sk_cdata.sk_rx_map[i]);
+ if (error != 0) {
+ printf(": unable to create rx DMA map %d, "
+    "error = %d\n", i, error);
+ goto fail_4;
+ }
  }
 
  ifp = &sc_if->arpcom.ac_if;
@@ -1197,12 +1057,18 @@ sk_attach(struct device *parent, struct
 
  DPRINTFN(2, ("sk_attach: end\n"));
  return;
+fail_4:
+ for (i = 0; i < SK_RX_RING_CNT; i++) {
+ if (sc_if->sk_cdata.sk_rx_map[i] == NULL)
+ continue;
 
-fail_2:
+ bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_map[i]);
+ }
+fail_3:
  bus_dmamem_unmap(sc->sc_dmatag, kva, sizeof(struct sk_ring_data));
-fail_1:
+fail_2:
  bus_dmamem_free(sc->sc_dmatag, &sc_if->sk_ring_seg, sc_if->sk_ring_nseg);
-fail_3:
+fail_1:
  bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_ring_map);
 fail:
  sc->sk_if[sa->skc_port] = NULL;
@@ -1722,45 +1588,44 @@ sk_rxeof(struct sk_if_softc *sc_if)
 {
  struct sk_softc *sc = sc_if->sk_softc;
  struct ifnet *ifp = &sc_if->arpcom.ac_if;
+ struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
  struct mbuf *m;
  struct sk_chain *cur_rx;
  struct sk_rx_desc *cur_desc;
- int i, cur, total_len = 0;
+ int cur, total_len = 0;
  u_int32_t rxstat, sk_ctl;
  bus_dmamap_t dmamap;
 
  DPRINTFN(2, ("sk_rxeof\n"));
 
- i = sc_if->sk_cdata.sk_rx_prod;
-
- for (;;) {
- cur = i;
+ while (if_rxr_inuse(rxr) > 0) {
+ cur = sc_if->sk_cdata.sk_rx_cons;
 
  /* Sync the descriptor */
  SK_CDRXSYNC(sc_if, cur,
     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 
- sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[i].sk_ctl);
- if ((sk_ctl & SK_RXCTL_OWN) != 0) {
- /* Invalidate the descriptor -- it's not ready yet */
- SK_CDRXSYNC(sc_if, cur, BUS_DMASYNC_PREREAD);
- sc_if->sk_cdata.sk_rx_prod = i;
+ cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
+ if (cur_rx->sk_mbuf == NULL)
+ break;
+
+ sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[cur].sk_ctl);
+ if ((sk_ctl & SK_RXCTL_OWN) != 0)
  break;
- }
 
- cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
  cur_desc = &sc_if->sk_rdata->sk_rx_ring[cur];
- dmamap = sc_if->sk_cdata.sk_rx_jumbo_map;
+ dmamap = sc_if->sk_cdata.sk_rx_map[cur];
 
  bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
     dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
 
- rxstat = letoh32(cur_desc->sk_xmac_rxstat);
  m = cur_rx->sk_mbuf;
  cur_rx->sk_mbuf = NULL;
- total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
+ if_rxr_put(rxr, 1);
 
- SK_INC(i, SK_RX_RING_CNT);
+ total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
+ rxstat = letoh32(cur_desc->sk_xmac_rxstat);
 
  if ((sk_ctl & (SK_RXCTL_STATUS_VALID | SK_RXCTL_FIRSTFRAG |
     SK_RXCTL_LASTFRAG)) != (SK_RXCTL_STATUS_VALID |
@@ -1769,31 +1634,12 @@ sk_rxeof(struct sk_if_softc *sc_if)
     total_len > SK_JUMBO_FRAMELEN ||
     sk_rxvalid(sc, rxstat, total_len) == 0) {
  ifp->if_ierrors++;
- sk_newbuf(sc_if, cur, m, dmamap);
+ m_freem(m);
  continue;
  }
 
- /*
- * Try to allocate a new jumbo buffer. If that
- * fails, copy the packet to mbufs and put the
- * jumbo buffer back in the ring so it can be
- * re-used. If allocating mbufs fails, then we
- * have to drop the packet.
- */
- if (sk_newbuf(sc_if, cur, NULL, dmamap) == ENOBUFS) {
- struct mbuf *m0;
- m0 = m_devget(mtod(m, char *), total_len, ETHER_ALIGN,
-    ifp);
- sk_newbuf(sc_if, cur, m, dmamap);
- if (m0 == NULL) {
- ifp->if_ierrors++;
- continue;
- }
- m = m0;
- } else {
- m->m_pkthdr.rcvif = ifp;
- m->m_pkthdr.len = m->m_len = total_len;
- }
+ m->m_pkthdr.rcvif = ifp;
+ m->m_pkthdr.len = m->m_len = total_len;
 
  ifp->if_ipackets++;
 
@@ -1805,6 +1651,8 @@ sk_rxeof(struct sk_if_softc *sc_if)
  /* pass it on. */
  ether_input_mbuf(ifp, m);
  }
+
+ sk_fill_rx_ring(sc_if);
 }
 
 void
@@ -2574,6 +2422,7 @@ sk_stop(struct sk_if_softc *sc_if, int s
 {
  struct sk_softc *sc = sc_if->sk_softc;
  struct ifnet *ifp = &sc_if->arpcom.ac_if;
+ bus_dmamap_t dmamap;
  struct sk_txmap_entry *dma;
  int i;
  u_int32_t val;
@@ -2664,6 +2513,10 @@ sk_stop(struct sk_if_softc *sc_if, int s
  /* Free RX and TX mbufs still in the queues. */
  for (i = 0; i < SK_RX_RING_CNT; i++) {
  if (sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf != NULL) {
+ dmamap = sc_if->sk_cdata.sk_rx_map[i];
+ bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
+    dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
  m_freem(sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf);
  sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf = NULL;
  }
Index: if_skvar.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_skvar.h,v
retrieving revision 1.8
diff -u -p -r1.8 if_skvar.h
--- if_skvar.h 7 Mar 2013 06:13:31 -0000 1.8
+++ if_skvar.h 12 Jul 2014 08:29:20 -0000
@@ -81,11 +81,6 @@
 #ifndef _DEV_PCI_IF_SKVAR_H_
 #define _DEV_PCI_IF_SKVAR_H_
 
-struct sk_jpool_entry {
- int                             slot;
- LIST_ENTRY(sk_jpool_entry) jpool_entries;
-};
-
 struct sk_chain {
  void *sk_desc;
  struct mbuf *sk_mbuf;
@@ -111,17 +106,12 @@ struct sk_chain_data {
  struct sk_chain sk_rx_chain[SK_RX_RING_CNT];
  struct sk_txmap_entry *sk_tx_map[SK_TX_RING_CNT];
  bus_dmamap_t sk_rx_map[SK_RX_RING_CNT];
- bus_dmamap_t sk_rx_jumbo_map;
  int sk_tx_prod;
  int sk_tx_cons;
  int sk_tx_cnt;
  int sk_rx_prod;
  int sk_rx_cons;
- int sk_rx_cnt;
- /* Stick the jumbo mem management stuff here too. */
- caddr_t sk_jslots[SK_JSLOTS];
- void *sk_jumbo_buf;
-
+ struct if_rxring sk_rx_ring;
 };
 
 struct sk_ring_data {
@@ -223,8 +213,6 @@ struct sk_if_softc {
  int sk_ring_nseg;
  struct sk_softc *sk_softc; /* parent controller */
  int sk_tx_bmu; /* TX BMU register */
- LIST_HEAD(__sk_jfreehead, sk_jpool_entry) sk_jfree_listhead;
- LIST_HEAD(__sk_jinusehead, sk_jpool_entry) sk_jinuse_listhead;
  SIMPLEQ_HEAD(__sk_txmaphead, sk_txmap_entry) sk_txmap_head;
 };
 

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Brad Smith-14
On 12/07/14 4:32 AM, David Gwynne wrote:
> how about this?

Now it attaches without error but tcpdump shows no traffic coming in
at all and there is regular traffic on the segment from spanning tree,
CARP, RA, etc.

$ vmstat -iz
interrupt                       total     rate
schizo0:pci_a                       0        0
schizo0:ue                          0        0
schizo0:ce                          0        0
schizo0:safari                      0        0
pcfiic0                             0        0
power0                              0        0
com0                               53        0
com1                                0        0
autri0                              0        0
ohci0                               0        0
ohci1                               0        0
pciide0                          1318        4
ohci2                               0        0
ohci3                               0        0
ehci0                               0        0

skc0                                5        0

schizo1:pci_b                       0        0
schizo1:ue                          0        0
schizo1:ce                          0        0
schizo1:safari                      0        0
bge0                              969        3
clock                           30714       99
Total                           33059      107


--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
i think i'll try to find the sk at work and wire it up. its just annoying cos im pretty sure its sr optics with sc connectors.

thanks for testing.

On 13 Jul 2014, at 4:45 am, Brad Smith <[hidden email]> wrote:

> On 12/07/14 4:32 AM, David Gwynne wrote:
>> how about this?
>
> Now it attaches without error but tcpdump shows no traffic coming in
> at all and there is regular traffic on the segment from spanning tree,
> CARP, RA, etc.
>
> $ vmstat -iz
> interrupt                       total     rate
> schizo0:pci_a                       0        0
> schizo0:ue                          0        0
> schizo0:ce                          0        0
> schizo0:safari                      0        0
> pcfiic0                             0        0
> power0                              0        0
> com0                               53        0
> com1                                0        0
> autri0                              0        0
> ohci0                               0        0
> ohci1                               0        0
> pciide0                          1318        4
> ohci2                               0        0
> ohci3                               0        0
> ehci0                               0        0
>
> skc0                                5        0
>
> schizo1:pci_b                       0        0
> schizo1:ue                          0        0
> schizo1:ce                          0        0
> schizo1:safari                      0        0
> bge0                              969        3
> clock                           30714       99
> Total                           33059      107
>
>
> --
> This message has been scanned for viruses and
> dangerous content by MailScanner, and is
> believed to be clean.
>


Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
> i think i'll try to find the sk at work and wire it up. its just annoying cos im pretty sure its sr optics with sc connectors.
>
> thanks for testing.

how's this one?

Index: if_sk.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_sk.c,v
retrieving revision 1.170
diff -u -p -r1.170 if_sk.c
--- if_sk.c 22 Jul 2014 13:12:11 -0000 1.170
+++ if_sk.c 18 Aug 2014 21:57:14 -0000
@@ -153,12 +153,10 @@ void sk_watchdog(struct ifnet *);
 int sk_ifmedia_upd(struct ifnet *);
 void sk_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 void skc_reset(struct sk_softc *);
-int sk_newbuf(struct sk_if_softc *, int, struct mbuf *, bus_dmamap_t);
-int sk_alloc_jumbo_mem(struct sk_if_softc *);
-void *sk_jalloc(struct sk_if_softc *);
-void sk_jfree(caddr_t, u_int, void *);
+int sk_newbuf(struct sk_if_softc *);
 int sk_reset(struct sk_if_softc *);
 int sk_init_rx_ring(struct sk_if_softc *);
+void sk_fill_rx_ring(struct sk_if_softc *);
 int sk_init_tx_ring(struct sk_if_softc *);
 
 int sk_xmac_miibus_readreg(struct device *, int, int);
@@ -547,21 +545,29 @@ sk_init_rx_ring(struct sk_if_softc *sc_i
  rd->sk_rx_ring[i].sk_next = htole32(SK_RX_RING_ADDR(sc_if, nexti));
  }
 
- for (i = 0; i < SK_RX_RING_CNT; i++) {
- if (sk_newbuf(sc_if, i, NULL,
-    sc_if->sk_cdata.sk_rx_jumbo_map) == ENOBUFS) {
- printf("%s: failed alloc of %dth mbuf\n",
-    sc_if->sk_dev.dv_xname, i);
- return (ENOBUFS);
- }
- }
-
  sc_if->sk_cdata.sk_rx_prod = 0;
  sc_if->sk_cdata.sk_rx_cons = 0;
 
+ if_rxr_init(&sc_if->sk_cdata.sk_rx_ring, 2, SK_RX_RING_CNT);
+
+ sk_fill_rx_ring(sc_if);
+
  return (0);
 }
 
+void
+sk_fill_rx_ring(struct sk_if_softc *sc_if)
+{
+ struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
+ u_int slots;
+
+ for (slots = if_rxr_get(rxr, SK_RX_RING_CNT); slots > 0; slots--) {
+ if (sk_newbuf(sc_if) == ENOBUFS)
+ break;
+ }
+ if_rxr_put(rxr, slots);
+}
+
 int
 sk_init_tx_ring(struct sk_if_softc *sc_if)
 {
@@ -609,199 +615,49 @@ sk_init_tx_ring(struct sk_if_softc *sc_i
 }
 
 int
-sk_newbuf(struct sk_if_softc *sc_if, int i, struct mbuf *m,
-  bus_dmamap_t dmamap)
+sk_newbuf(struct sk_if_softc *sc_if)
 {
- struct mbuf *m_new = NULL;
+ struct mbuf *m;
  struct sk_chain *c;
  struct sk_rx_desc *r;
+ bus_dmamap_t dmamap;
+ u_int prod;
+ int error;
 
- if (m == NULL) {
- caddr_t buf = NULL;
-
- MGETHDR(m_new, M_DONTWAIT, MT_DATA);
- if (m_new == NULL)
- return (ENOBUFS);
-
- /* Allocate the jumbo buffer */
- buf = sk_jalloc(sc_if);
- if (buf == NULL) {
- m_freem(m_new);
- DPRINTFN(1, ("%s jumbo allocation failed -- packet "
-    "dropped!\n", sc_if->arpcom.ac_if.if_xname));
- return (ENOBUFS);
- }
-
- /* Attach the buffer to the mbuf */
- m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
- MEXTADD(m_new, buf, SK_JLEN, 0, sk_jfree, sc_if);
- } else {
- /*
- * We're re-using a previously allocated mbuf;
- * be sure to re-init pointers and lengths to
- * default values.
- */
- m_new = m;
- m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
- m_new->m_data = m_new->m_ext.ext_buf;
- }
- m_adj(m_new, ETHER_ALIGN);
-
- c = &sc_if->sk_cdata.sk_rx_chain[i];
- r = c->sk_desc;
- c->sk_mbuf = m_new;
- r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr +
-    (((vaddr_t)m_new->m_data
-             - (vaddr_t)sc_if->sk_cdata.sk_jumbo_buf)));
- r->sk_ctl = htole32(SK_JLEN | SK_RXSTAT);
-
- SK_CDRXSYNC(sc_if, i, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
+ m = MCLGETI(NULL, M_DONTWAIT, NULL, SK_JLEN);
+ if (m == NULL)
+ return (ENOBUFS);
 
- return (0);
-}
+        m->m_len = m->m_pkthdr.len = SK_JLEN;
+ m_adj(m, ETHER_ALIGN);
 
-/*
- * Memory management for jumbo frames.
- */
+ prod = sc_if->sk_cdata.sk_rx_prod;
+ dmamap = sc_if->sk_cdata.sk_rx_map[prod];
 
-int
-sk_alloc_jumbo_mem(struct sk_if_softc *sc_if)
-{
- struct sk_softc *sc = sc_if->sk_softc;
- caddr_t ptr, kva;
- bus_dma_segment_t seg;
- int i, rseg, state, error;
- struct sk_jpool_entry   *entry;
-
- state = error = 0;
-
- /* Grab a big chunk o' storage. */
- if (bus_dmamem_alloc(sc->sc_dmatag, SK_JMEM, PAGE_SIZE, 0,
-     &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
- printf(": can't alloc rx buffers");
+ error = bus_dmamap_load_mbuf(sc_if->sk_softc->sc_dmatag, dmamap, m,
+    BUS_DMA_READ|BUS_DMA_NOWAIT);
+ if (error) {
+ m_freem(m);
  return (ENOBUFS);
  }
 
- state = 1;
- if (bus_dmamem_map(sc->sc_dmatag, &seg, rseg, SK_JMEM, &kva,
-   BUS_DMA_NOWAIT)) {
- printf(": can't map dma buffers (%d bytes)", SK_JMEM);
- error = ENOBUFS;
- goto out;
- }
-
- state = 2;
- if (bus_dmamap_create(sc->sc_dmatag, SK_JMEM, 1, SK_JMEM, 0,
-    BUS_DMA_NOWAIT, &sc_if->sk_cdata.sk_rx_jumbo_map)) {
- printf(": can't create dma map");
- error = ENOBUFS;
- goto out;
- }
-
- state = 3;
- if (bus_dmamap_load(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_jumbo_map,
-    kva, SK_JMEM, NULL, BUS_DMA_NOWAIT)) {
- printf(": can't load dma map");
- error = ENOBUFS;
- goto out;
- }
-
- state = 4;
- sc_if->sk_cdata.sk_jumbo_buf = (caddr_t)kva;
- DPRINTFN(1,("sk_jumbo_buf = 0x%08X\n", sc_if->sk_cdata.sk_jumbo_buf));
+ bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
+    dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
 
- LIST_INIT(&sc_if->sk_jfree_listhead);
- LIST_INIT(&sc_if->sk_jinuse_listhead);
+ c = &sc_if->sk_cdata.sk_rx_chain[prod];
+ c->sk_mbuf = m;
 
- /*
- * Now divide it up into 9K pieces and save the addresses
- * in an array.
- */
- ptr = sc_if->sk_cdata.sk_jumbo_buf;
- for (i = 0; i < SK_JSLOTS; i++) {
- sc_if->sk_cdata.sk_jslots[i] = ptr;
- ptr += SK_JLEN;
- entry = malloc(sizeof(struct sk_jpool_entry),
-    M_DEVBUF, M_NOWAIT);
- if (entry == NULL) {
- sc_if->sk_cdata.sk_jumbo_buf = NULL;
- printf(": no memory for jumbo buffer queue!");
- error = ENOBUFS;
- goto out;
- }
- entry->slot = i;
- LIST_INSERT_HEAD(&sc_if->sk_jfree_listhead,
- entry, jpool_entries);
- }
-out:
- if (error != 0) {
- switch (state) {
- case 4:
- bus_dmamap_unload(sc->sc_dmatag,
-    sc_if->sk_cdata.sk_rx_jumbo_map);
- case 3:
- bus_dmamap_destroy(sc->sc_dmatag,
-    sc_if->sk_cdata.sk_rx_jumbo_map);
- case 2:
- bus_dmamem_unmap(sc->sc_dmatag, kva, SK_JMEM);
- case 1:
- bus_dmamem_free(sc->sc_dmatag, &seg, rseg);
- break;
- default:
- break;
- }
- }
-
- return (error);
-}
-
-/*
- * Allocate a jumbo buffer.
- */
-void *
-sk_jalloc(struct sk_if_softc *sc_if)
-{
- struct sk_jpool_entry   *entry;
-
- entry = LIST_FIRST(&sc_if->sk_jfree_listhead);
-
- if (entry == NULL)
- return (NULL);
-
- LIST_REMOVE(entry, jpool_entries);
- LIST_INSERT_HEAD(&sc_if->sk_jinuse_listhead, entry, jpool_entries);
- return (sc_if->sk_cdata.sk_jslots[entry->slot]);
-}
-
-/*
- * Release a jumbo buffer.
- */
-void
-sk_jfree(caddr_t buf, u_int size, void *arg)
-{
- struct sk_jpool_entry *entry;
- struct sk_if_softc *sc;
- int i;
+ r = c->sk_desc;
+ r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr);
+ r->sk_data_hi = htole32(((u_int64_t)dmamap->dm_segs[0].ds_addr) >> 32);
+ r->sk_ctl = htole32(dmamap->dm_segs[0].ds_len | SK_RXSTAT);
 
- /* Extract the softc struct pointer. */
- sc = (struct sk_if_softc *)arg;
+ SK_CDRXSYNC(sc_if, prod, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
 
- if (sc == NULL)
- panic("sk_jfree: can't find softc pointer!");
+ SK_INC(prod, SK_RX_RING_CNT);
+ sc_if->sk_cdata.sk_rx_prod = prod;
 
- /* calculate the slot this buffer belongs to */
- i = ((vaddr_t)buf
-     - (vaddr_t)sc->sk_cdata.sk_jumbo_buf) / SK_JLEN;
-
- if ((i < 0) || (i >= SK_JSLOTS))
- panic("sk_jfree: asked to free buffer that we don't manage!");
-
- entry = LIST_FIRST(&sc->sk_jinuse_listhead);
- if (entry == NULL)
- panic("sk_jfree: buffer not in use!");
- entry->slot = i;
- LIST_REMOVE(entry, jpool_entries);
- LIST_INSERT_HEAD(&sc->sk_jfree_listhead, entry, jpool_entries);
+ return (0);
 }
 
 /*
@@ -869,6 +725,12 @@ sk_ioctl(struct ifnet *ifp, u_long comma
  error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
  break;
 
+ case SIOCGIFRXR:
+ error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
+    NULL, SK_JLEN, &sc_if->sk_cdata.sk_rx_ring);
+
+ break;
+
  default:
  error = ether_ioctl(ifp, &sc_if->arpcom, command, data);
  }
@@ -997,7 +859,7 @@ sk_attach(struct device *parent, struct
  struct skc_attach_args *sa = aux;
  struct ifnet *ifp;
  caddr_t kva;
- int i;
+ int i, error;
 
  sc_if->sk_port = sa->skc_port;
  sc_if->sk_softc = sc;
@@ -1124,10 +986,14 @@ sk_attach(struct device *parent, struct
  }
         sc_if->sk_rdata = (struct sk_ring_data *)kva;
 
- /* Try to allocate memory for jumbo buffers. */
- if (sk_alloc_jumbo_mem(sc_if)) {
- printf(": jumbo buffer allocation failed\n");
- goto fail_3;
+ for (i = 0; i < SK_RX_RING_CNT; i++) {
+ error = bus_dmamap_create(sc->sc_dmatag, SK_JLEN, 1,
+    SK_JLEN, 0, 0, &sc_if->sk_cdata.sk_rx_map[i]);
+ if (error != 0) {
+ printf(": unable to create rx DMA map %d, "
+    "error = %d\n", i, error);
+ goto fail_4;
+ }
  }
 
  ifp = &sc_if->arpcom.ac_if;
@@ -1193,12 +1059,18 @@ sk_attach(struct device *parent, struct
 
  DPRINTFN(2, ("sk_attach: end\n"));
  return;
+fail_4:
+ for (i = 0; i < SK_RX_RING_CNT; i++) {
+ if (sc_if->sk_cdata.sk_rx_map[i] == NULL)
+ continue;
 
-fail_2:
+ bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_map[i]);
+ }
+fail_3:
  bus_dmamem_unmap(sc->sc_dmatag, kva, sizeof(struct sk_ring_data));
-fail_1:
+fail_2:
  bus_dmamem_free(sc->sc_dmatag, &sc_if->sk_ring_seg, sc_if->sk_ring_nseg);
-fail_3:
+fail_1:
  bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_ring_map);
 fail:
  sc->sk_if[sa->skc_port] = NULL;
@@ -1718,45 +1590,44 @@ sk_rxeof(struct sk_if_softc *sc_if)
 {
  struct sk_softc *sc = sc_if->sk_softc;
  struct ifnet *ifp = &sc_if->arpcom.ac_if;
+ struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
  struct mbuf *m;
  struct sk_chain *cur_rx;
  struct sk_rx_desc *cur_desc;
- int i, cur, total_len = 0;
+ int cur, total_len = 0;
  u_int32_t rxstat, sk_ctl;
  bus_dmamap_t dmamap;
 
  DPRINTFN(2, ("sk_rxeof\n"));
 
- i = sc_if->sk_cdata.sk_rx_prod;
-
- for (;;) {
- cur = i;
-
+ cur = sc_if->sk_cdata.sk_rx_cons;
+ while (if_rxr_inuse(rxr) > 0) {
  /* Sync the descriptor */
  SK_CDRXSYNC(sc_if, cur,
     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 
- sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[i].sk_ctl);
- if ((sk_ctl & SK_RXCTL_OWN) != 0) {
- /* Invalidate the descriptor -- it's not ready yet */
- SK_CDRXSYNC(sc_if, cur, BUS_DMASYNC_PREREAD);
- sc_if->sk_cdata.sk_rx_prod = i;
+ cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
+ if (cur_rx->sk_mbuf == NULL)
+ break;
+
+ sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[cur].sk_ctl);
+ if ((sk_ctl & SK_RXCTL_OWN) != 0)
  break;
- }
 
- cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
  cur_desc = &sc_if->sk_rdata->sk_rx_ring[cur];
- dmamap = sc_if->sk_cdata.sk_rx_jumbo_map;
+ dmamap = sc_if->sk_cdata.sk_rx_map[cur];
 
  bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
     dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
 
- rxstat = letoh32(cur_desc->sk_xmac_rxstat);
  m = cur_rx->sk_mbuf;
  cur_rx->sk_mbuf = NULL;
- total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
+ if_rxr_put(rxr, 1);
+ SK_INC(cur, SK_RX_RING_CNT);
 
- SK_INC(i, SK_RX_RING_CNT);
+ total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
+ rxstat = letoh32(cur_desc->sk_xmac_rxstat);
 
  if ((sk_ctl & (SK_RXCTL_STATUS_VALID | SK_RXCTL_FIRSTFRAG |
     SK_RXCTL_LASTFRAG)) != (SK_RXCTL_STATUS_VALID |
@@ -1765,31 +1636,12 @@ sk_rxeof(struct sk_if_softc *sc_if)
     total_len > SK_JUMBO_FRAMELEN ||
     sk_rxvalid(sc, rxstat, total_len) == 0) {
  ifp->if_ierrors++;
- sk_newbuf(sc_if, cur, m, dmamap);
+ m_freem(m);
  continue;
  }
 
- /*
- * Try to allocate a new jumbo buffer. If that
- * fails, copy the packet to mbufs and put the
- * jumbo buffer back in the ring so it can be
- * re-used. If allocating mbufs fails, then we
- * have to drop the packet.
- */
- if (sk_newbuf(sc_if, cur, NULL, dmamap) == ENOBUFS) {
- struct mbuf *m0;
- m0 = m_devget(mtod(m, char *), total_len, ETHER_ALIGN,
-    ifp);
- sk_newbuf(sc_if, cur, m, dmamap);
- if (m0 == NULL) {
- ifp->if_ierrors++;
- continue;
- }
- m = m0;
- } else {
- m->m_pkthdr.rcvif = ifp;
- m->m_pkthdr.len = m->m_len = total_len;
- }
+ m->m_pkthdr.rcvif = ifp;
+ m->m_pkthdr.len = m->m_len = total_len;
 
  ifp->if_ipackets++;
 
@@ -1801,6 +1653,9 @@ sk_rxeof(struct sk_if_softc *sc_if)
  /* pass it on. */
  ether_input_mbuf(ifp, m);
  }
+ sc_if->sk_cdata.sk_rx_cons = cur;
+
+ sk_fill_rx_ring(sc_if);
 }
 
 void
@@ -2570,6 +2425,7 @@ sk_stop(struct sk_if_softc *sc_if, int s
 {
  struct sk_softc *sc = sc_if->sk_softc;
  struct ifnet *ifp = &sc_if->arpcom.ac_if;
+ bus_dmamap_t dmamap;
  struct sk_txmap_entry *dma;
  int i;
  u_int32_t val;
@@ -2660,6 +2516,10 @@ sk_stop(struct sk_if_softc *sc_if, int s
  /* Free RX and TX mbufs still in the queues. */
  for (i = 0; i < SK_RX_RING_CNT; i++) {
  if (sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf != NULL) {
+ dmamap = sc_if->sk_cdata.sk_rx_map[i];
+ bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
+    dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
  m_freem(sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf);
  sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf = NULL;
  }
Index: if_skvar.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_skvar.h,v
retrieving revision 1.8
diff -u -p -r1.8 if_skvar.h
--- if_skvar.h 7 Mar 2013 06:13:31 -0000 1.8
+++ if_skvar.h 18 Aug 2014 21:57:14 -0000
@@ -81,11 +81,6 @@
 #ifndef _DEV_PCI_IF_SKVAR_H_
 #define _DEV_PCI_IF_SKVAR_H_
 
-struct sk_jpool_entry {
- int                             slot;
- LIST_ENTRY(sk_jpool_entry) jpool_entries;
-};
-
 struct sk_chain {
  void *sk_desc;
  struct mbuf *sk_mbuf;
@@ -111,17 +106,12 @@ struct sk_chain_data {
  struct sk_chain sk_rx_chain[SK_RX_RING_CNT];
  struct sk_txmap_entry *sk_tx_map[SK_TX_RING_CNT];
  bus_dmamap_t sk_rx_map[SK_RX_RING_CNT];
- bus_dmamap_t sk_rx_jumbo_map;
  int sk_tx_prod;
  int sk_tx_cons;
  int sk_tx_cnt;
  int sk_rx_prod;
  int sk_rx_cons;
- int sk_rx_cnt;
- /* Stick the jumbo mem management stuff here too. */
- caddr_t sk_jslots[SK_JSLOTS];
- void *sk_jumbo_buf;
-
+ struct if_rxring sk_rx_ring;
 };
 
 struct sk_ring_data {
@@ -223,8 +213,6 @@ struct sk_if_softc {
  int sk_ring_nseg;
  struct sk_softc *sk_softc; /* parent controller */
  int sk_tx_bmu; /* TX BMU register */
- LIST_HEAD(__sk_jfreehead, sk_jpool_entry) sk_jfree_listhead;
- LIST_HEAD(__sk_jinusehead, sk_jpool_entry) sk_jinuse_listhead;
  SIMPLEQ_HEAD(__sk_txmaphead, sk_txmap_entry) sk_txmap_head;
 };
 

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Brad Smith-14
On 18/08/14 6:24 PM, David Gwynne wrote:
> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>> i think i'll try to find the sk at work and wire it up. its just annoying cos im pretty sure its sr optics with sc connectors.
>>
>> thanks for testing.
>
> how's this one?

I'll look into testing it later on tonight or tomorrow.

> Index: if_sk.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/if_sk.c,v
> retrieving revision 1.170
> diff -u -p -r1.170 if_sk.c
> --- if_sk.c 22 Jul 2014 13:12:11 -0000 1.170
> +++ if_sk.c 18 Aug 2014 21:57:14 -0000
> @@ -153,12 +153,10 @@ void sk_watchdog(struct ifnet *);
>   int sk_ifmedia_upd(struct ifnet *);
>   void sk_ifmedia_sts(struct ifnet *, struct ifmediareq *);
>   void skc_reset(struct sk_softc *);
> -int sk_newbuf(struct sk_if_softc *, int, struct mbuf *, bus_dmamap_t);
> -int sk_alloc_jumbo_mem(struct sk_if_softc *);
> -void *sk_jalloc(struct sk_if_softc *);
> -void sk_jfree(caddr_t, u_int, void *);
> +int sk_newbuf(struct sk_if_softc *);
>   int sk_reset(struct sk_if_softc *);
>   int sk_init_rx_ring(struct sk_if_softc *);
> +void sk_fill_rx_ring(struct sk_if_softc *);
>   int sk_init_tx_ring(struct sk_if_softc *);
>
>   int sk_xmac_miibus_readreg(struct device *, int, int);
> @@ -547,21 +545,29 @@ sk_init_rx_ring(struct sk_if_softc *sc_i
>   rd->sk_rx_ring[i].sk_next = htole32(SK_RX_RING_ADDR(sc_if, nexti));
>   }
>
> - for (i = 0; i < SK_RX_RING_CNT; i++) {
> - if (sk_newbuf(sc_if, i, NULL,
> -    sc_if->sk_cdata.sk_rx_jumbo_map) == ENOBUFS) {
> - printf("%s: failed alloc of %dth mbuf\n",
> -    sc_if->sk_dev.dv_xname, i);
> - return (ENOBUFS);
> - }
> - }
> -
>   sc_if->sk_cdata.sk_rx_prod = 0;
>   sc_if->sk_cdata.sk_rx_cons = 0;
>
> + if_rxr_init(&sc_if->sk_cdata.sk_rx_ring, 2, SK_RX_RING_CNT);
> +
> + sk_fill_rx_ring(sc_if);
> +
>   return (0);
>   }
>
> +void
> +sk_fill_rx_ring(struct sk_if_softc *sc_if)
> +{
> + struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
> + u_int slots;
> +
> + for (slots = if_rxr_get(rxr, SK_RX_RING_CNT); slots > 0; slots--) {
> + if (sk_newbuf(sc_if) == ENOBUFS)
> + break;
> + }
> + if_rxr_put(rxr, slots);
> +}
> +
>   int
>   sk_init_tx_ring(struct sk_if_softc *sc_if)
>   {
> @@ -609,199 +615,49 @@ sk_init_tx_ring(struct sk_if_softc *sc_i
>   }
>
>   int
> -sk_newbuf(struct sk_if_softc *sc_if, int i, struct mbuf *m,
> -  bus_dmamap_t dmamap)
> +sk_newbuf(struct sk_if_softc *sc_if)
>   {
> - struct mbuf *m_new = NULL;
> + struct mbuf *m;
>   struct sk_chain *c;
>   struct sk_rx_desc *r;
> + bus_dmamap_t dmamap;
> + u_int prod;
> + int error;
>
> - if (m == NULL) {
> - caddr_t buf = NULL;
> -
> - MGETHDR(m_new, M_DONTWAIT, MT_DATA);
> - if (m_new == NULL)
> - return (ENOBUFS);
> -
> - /* Allocate the jumbo buffer */
> - buf = sk_jalloc(sc_if);
> - if (buf == NULL) {
> - m_freem(m_new);
> - DPRINTFN(1, ("%s jumbo allocation failed -- packet "
> -    "dropped!\n", sc_if->arpcom.ac_if.if_xname));
> - return (ENOBUFS);
> - }
> -
> - /* Attach the buffer to the mbuf */
> - m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
> - MEXTADD(m_new, buf, SK_JLEN, 0, sk_jfree, sc_if);
> - } else {
> - /*
> - * We're re-using a previously allocated mbuf;
> - * be sure to re-init pointers and lengths to
> - * default values.
> - */
> - m_new = m;
> - m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
> - m_new->m_data = m_new->m_ext.ext_buf;
> - }
> - m_adj(m_new, ETHER_ALIGN);
> -
> - c = &sc_if->sk_cdata.sk_rx_chain[i];
> - r = c->sk_desc;
> - c->sk_mbuf = m_new;
> - r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr +
> -    (((vaddr_t)m_new->m_data
> -             - (vaddr_t)sc_if->sk_cdata.sk_jumbo_buf)));
> - r->sk_ctl = htole32(SK_JLEN | SK_RXSTAT);
> -
> - SK_CDRXSYNC(sc_if, i, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
> + m = MCLGETI(NULL, M_DONTWAIT, NULL, SK_JLEN);
> + if (m == NULL)
> + return (ENOBUFS);
>
> - return (0);
> -}
> +        m->m_len = m->m_pkthdr.len = SK_JLEN;
> + m_adj(m, ETHER_ALIGN);
>
> -/*
> - * Memory management for jumbo frames.
> - */
> + prod = sc_if->sk_cdata.sk_rx_prod;
> + dmamap = sc_if->sk_cdata.sk_rx_map[prod];
>
> -int
> -sk_alloc_jumbo_mem(struct sk_if_softc *sc_if)
> -{
> - struct sk_softc *sc = sc_if->sk_softc;
> - caddr_t ptr, kva;
> - bus_dma_segment_t seg;
> - int i, rseg, state, error;
> - struct sk_jpool_entry   *entry;
> -
> - state = error = 0;
> -
> - /* Grab a big chunk o' storage. */
> - if (bus_dmamem_alloc(sc->sc_dmatag, SK_JMEM, PAGE_SIZE, 0,
> -     &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
> - printf(": can't alloc rx buffers");
> + error = bus_dmamap_load_mbuf(sc_if->sk_softc->sc_dmatag, dmamap, m,
> +    BUS_DMA_READ|BUS_DMA_NOWAIT);
> + if (error) {
> + m_freem(m);
>   return (ENOBUFS);
>   }
>
> - state = 1;
> - if (bus_dmamem_map(sc->sc_dmatag, &seg, rseg, SK_JMEM, &kva,
> -   BUS_DMA_NOWAIT)) {
> - printf(": can't map dma buffers (%d bytes)", SK_JMEM);
> - error = ENOBUFS;
> - goto out;
> - }
> -
> - state = 2;
> - if (bus_dmamap_create(sc->sc_dmatag, SK_JMEM, 1, SK_JMEM, 0,
> -    BUS_DMA_NOWAIT, &sc_if->sk_cdata.sk_rx_jumbo_map)) {
> - printf(": can't create dma map");
> - error = ENOBUFS;
> - goto out;
> - }
> -
> - state = 3;
> - if (bus_dmamap_load(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_jumbo_map,
> -    kva, SK_JMEM, NULL, BUS_DMA_NOWAIT)) {
> - printf(": can't load dma map");
> - error = ENOBUFS;
> - goto out;
> - }
> -
> - state = 4;
> - sc_if->sk_cdata.sk_jumbo_buf = (caddr_t)kva;
> - DPRINTFN(1,("sk_jumbo_buf = 0x%08X\n", sc_if->sk_cdata.sk_jumbo_buf));
> + bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
> +    dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
>
> - LIST_INIT(&sc_if->sk_jfree_listhead);
> - LIST_INIT(&sc_if->sk_jinuse_listhead);
> + c = &sc_if->sk_cdata.sk_rx_chain[prod];
> + c->sk_mbuf = m;
>
> - /*
> - * Now divide it up into 9K pieces and save the addresses
> - * in an array.
> - */
> - ptr = sc_if->sk_cdata.sk_jumbo_buf;
> - for (i = 0; i < SK_JSLOTS; i++) {
> - sc_if->sk_cdata.sk_jslots[i] = ptr;
> - ptr += SK_JLEN;
> - entry = malloc(sizeof(struct sk_jpool_entry),
> -    M_DEVBUF, M_NOWAIT);
> - if (entry == NULL) {
> - sc_if->sk_cdata.sk_jumbo_buf = NULL;
> - printf(": no memory for jumbo buffer queue!");
> - error = ENOBUFS;
> - goto out;
> - }
> - entry->slot = i;
> - LIST_INSERT_HEAD(&sc_if->sk_jfree_listhead,
> - entry, jpool_entries);
> - }
> -out:
> - if (error != 0) {
> - switch (state) {
> - case 4:
> - bus_dmamap_unload(sc->sc_dmatag,
> -    sc_if->sk_cdata.sk_rx_jumbo_map);
> - case 3:
> - bus_dmamap_destroy(sc->sc_dmatag,
> -    sc_if->sk_cdata.sk_rx_jumbo_map);
> - case 2:
> - bus_dmamem_unmap(sc->sc_dmatag, kva, SK_JMEM);
> - case 1:
> - bus_dmamem_free(sc->sc_dmatag, &seg, rseg);
> - break;
> - default:
> - break;
> - }
> - }
> -
> - return (error);
> -}
> -
> -/*
> - * Allocate a jumbo buffer.
> - */
> -void *
> -sk_jalloc(struct sk_if_softc *sc_if)
> -{
> - struct sk_jpool_entry   *entry;
> -
> - entry = LIST_FIRST(&sc_if->sk_jfree_listhead);
> -
> - if (entry == NULL)
> - return (NULL);
> -
> - LIST_REMOVE(entry, jpool_entries);
> - LIST_INSERT_HEAD(&sc_if->sk_jinuse_listhead, entry, jpool_entries);
> - return (sc_if->sk_cdata.sk_jslots[entry->slot]);
> -}
> -
> -/*
> - * Release a jumbo buffer.
> - */
> -void
> -sk_jfree(caddr_t buf, u_int size, void *arg)
> -{
> - struct sk_jpool_entry *entry;
> - struct sk_if_softc *sc;
> - int i;
> + r = c->sk_desc;
> + r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr);
> + r->sk_data_hi = htole32(((u_int64_t)dmamap->dm_segs[0].ds_addr) >> 32);
> + r->sk_ctl = htole32(dmamap->dm_segs[0].ds_len | SK_RXSTAT);
>
> - /* Extract the softc struct pointer. */
> - sc = (struct sk_if_softc *)arg;
> + SK_CDRXSYNC(sc_if, prod, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
>
> - if (sc == NULL)
> - panic("sk_jfree: can't find softc pointer!");
> + SK_INC(prod, SK_RX_RING_CNT);
> + sc_if->sk_cdata.sk_rx_prod = prod;
>
> - /* calculate the slot this buffer belongs to */
> - i = ((vaddr_t)buf
> -     - (vaddr_t)sc->sk_cdata.sk_jumbo_buf) / SK_JLEN;
> -
> - if ((i < 0) || (i >= SK_JSLOTS))
> - panic("sk_jfree: asked to free buffer that we don't manage!");
> -
> - entry = LIST_FIRST(&sc->sk_jinuse_listhead);
> - if (entry == NULL)
> - panic("sk_jfree: buffer not in use!");
> - entry->slot = i;
> - LIST_REMOVE(entry, jpool_entries);
> - LIST_INSERT_HEAD(&sc->sk_jfree_listhead, entry, jpool_entries);
> + return (0);
>   }
>
>   /*
> @@ -869,6 +725,12 @@ sk_ioctl(struct ifnet *ifp, u_long comma
>   error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
>   break;
>
> + case SIOCGIFRXR:
> + error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
> +    NULL, SK_JLEN, &sc_if->sk_cdata.sk_rx_ring);
> +
> + break;
> +
>   default:
>   error = ether_ioctl(ifp, &sc_if->arpcom, command, data);
>   }
> @@ -997,7 +859,7 @@ sk_attach(struct device *parent, struct
>   struct skc_attach_args *sa = aux;
>   struct ifnet *ifp;
>   caddr_t kva;
> - int i;
> + int i, error;
>
>   sc_if->sk_port = sa->skc_port;
>   sc_if->sk_softc = sc;
> @@ -1124,10 +986,14 @@ sk_attach(struct device *parent, struct
>   }
>           sc_if->sk_rdata = (struct sk_ring_data *)kva;
>
> - /* Try to allocate memory for jumbo buffers. */
> - if (sk_alloc_jumbo_mem(sc_if)) {
> - printf(": jumbo buffer allocation failed\n");
> - goto fail_3;
> + for (i = 0; i < SK_RX_RING_CNT; i++) {
> + error = bus_dmamap_create(sc->sc_dmatag, SK_JLEN, 1,
> +    SK_JLEN, 0, 0, &sc_if->sk_cdata.sk_rx_map[i]);
> + if (error != 0) {
> + printf(": unable to create rx DMA map %d, "
> +    "error = %d\n", i, error);
> + goto fail_4;
> + }
>   }
>
>   ifp = &sc_if->arpcom.ac_if;
> @@ -1193,12 +1059,18 @@ sk_attach(struct device *parent, struct
>
>   DPRINTFN(2, ("sk_attach: end\n"));
>   return;
> +fail_4:
> + for (i = 0; i < SK_RX_RING_CNT; i++) {
> + if (sc_if->sk_cdata.sk_rx_map[i] == NULL)
> + continue;
>
> -fail_2:
> + bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_map[i]);
> + }
> +fail_3:
>   bus_dmamem_unmap(sc->sc_dmatag, kva, sizeof(struct sk_ring_data));
> -fail_1:
> +fail_2:
>   bus_dmamem_free(sc->sc_dmatag, &sc_if->sk_ring_seg, sc_if->sk_ring_nseg);
> -fail_3:
> +fail_1:
>   bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_ring_map);
>   fail:
>   sc->sk_if[sa->skc_port] = NULL;
> @@ -1718,45 +1590,44 @@ sk_rxeof(struct sk_if_softc *sc_if)
>   {
>   struct sk_softc *sc = sc_if->sk_softc;
>   struct ifnet *ifp = &sc_if->arpcom.ac_if;
> + struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
>   struct mbuf *m;
>   struct sk_chain *cur_rx;
>   struct sk_rx_desc *cur_desc;
> - int i, cur, total_len = 0;
> + int cur, total_len = 0;
>   u_int32_t rxstat, sk_ctl;
>   bus_dmamap_t dmamap;
>
>   DPRINTFN(2, ("sk_rxeof\n"));
>
> - i = sc_if->sk_cdata.sk_rx_prod;
> -
> - for (;;) {
> - cur = i;
> -
> + cur = sc_if->sk_cdata.sk_rx_cons;
> + while (if_rxr_inuse(rxr) > 0) {
>   /* Sync the descriptor */
>   SK_CDRXSYNC(sc_if, cur,
>      BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
>
> - sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[i].sk_ctl);
> - if ((sk_ctl & SK_RXCTL_OWN) != 0) {
> - /* Invalidate the descriptor -- it's not ready yet */
> - SK_CDRXSYNC(sc_if, cur, BUS_DMASYNC_PREREAD);
> - sc_if->sk_cdata.sk_rx_prod = i;
> + cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
> + if (cur_rx->sk_mbuf == NULL)
> + break;
> +
> + sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[cur].sk_ctl);
> + if ((sk_ctl & SK_RXCTL_OWN) != 0)
>   break;
> - }
>
> - cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
>   cur_desc = &sc_if->sk_rdata->sk_rx_ring[cur];
> - dmamap = sc_if->sk_cdata.sk_rx_jumbo_map;
> + dmamap = sc_if->sk_cdata.sk_rx_map[cur];
>
>   bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
>      dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
> + bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
>
> - rxstat = letoh32(cur_desc->sk_xmac_rxstat);
>   m = cur_rx->sk_mbuf;
>   cur_rx->sk_mbuf = NULL;
> - total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
> + if_rxr_put(rxr, 1);
> + SK_INC(cur, SK_RX_RING_CNT);
>
> - SK_INC(i, SK_RX_RING_CNT);
> + total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
> + rxstat = letoh32(cur_desc->sk_xmac_rxstat);
>
>   if ((sk_ctl & (SK_RXCTL_STATUS_VALID | SK_RXCTL_FIRSTFRAG |
>      SK_RXCTL_LASTFRAG)) != (SK_RXCTL_STATUS_VALID |
> @@ -1765,31 +1636,12 @@ sk_rxeof(struct sk_if_softc *sc_if)
>      total_len > SK_JUMBO_FRAMELEN ||
>      sk_rxvalid(sc, rxstat, total_len) == 0) {
>   ifp->if_ierrors++;
> - sk_newbuf(sc_if, cur, m, dmamap);
> + m_freem(m);
>   continue;
>   }
>
> - /*
> - * Try to allocate a new jumbo buffer. If that
> - * fails, copy the packet to mbufs and put the
> - * jumbo buffer back in the ring so it can be
> - * re-used. If allocating mbufs fails, then we
> - * have to drop the packet.
> - */
> - if (sk_newbuf(sc_if, cur, NULL, dmamap) == ENOBUFS) {
> - struct mbuf *m0;
> - m0 = m_devget(mtod(m, char *), total_len, ETHER_ALIGN,
> -    ifp);
> - sk_newbuf(sc_if, cur, m, dmamap);
> - if (m0 == NULL) {
> - ifp->if_ierrors++;
> - continue;
> - }
> - m = m0;
> - } else {
> - m->m_pkthdr.rcvif = ifp;
> - m->m_pkthdr.len = m->m_len = total_len;
> - }
> + m->m_pkthdr.rcvif = ifp;
> + m->m_pkthdr.len = m->m_len = total_len;
>
>   ifp->if_ipackets++;
>
> @@ -1801,6 +1653,9 @@ sk_rxeof(struct sk_if_softc *sc_if)
>   /* pass it on. */
>   ether_input_mbuf(ifp, m);
>   }
> + sc_if->sk_cdata.sk_rx_cons = cur;
> +
> + sk_fill_rx_ring(sc_if);
>   }
>
>   void
> @@ -2570,6 +2425,7 @@ sk_stop(struct sk_if_softc *sc_if, int s
>   {
>   struct sk_softc *sc = sc_if->sk_softc;
>   struct ifnet *ifp = &sc_if->arpcom.ac_if;
> + bus_dmamap_t dmamap;
>   struct sk_txmap_entry *dma;
>   int i;
>   u_int32_t val;
> @@ -2660,6 +2516,10 @@ sk_stop(struct sk_if_softc *sc_if, int s
>   /* Free RX and TX mbufs still in the queues. */
>   for (i = 0; i < SK_RX_RING_CNT; i++) {
>   if (sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf != NULL) {
> + dmamap = sc_if->sk_cdata.sk_rx_map[i];
> + bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
> +    dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
> + bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
>   m_freem(sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf);
>   sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf = NULL;
>   }
> Index: if_skvar.h
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/if_skvar.h,v
> retrieving revision 1.8
> diff -u -p -r1.8 if_skvar.h
> --- if_skvar.h 7 Mar 2013 06:13:31 -0000 1.8
> +++ if_skvar.h 18 Aug 2014 21:57:14 -0000
> @@ -81,11 +81,6 @@
>   #ifndef _DEV_PCI_IF_SKVAR_H_
>   #define _DEV_PCI_IF_SKVAR_H_
>
> -struct sk_jpool_entry {
> - int                             slot;
> - LIST_ENTRY(sk_jpool_entry) jpool_entries;
> -};
> -
>   struct sk_chain {
>   void *sk_desc;
>   struct mbuf *sk_mbuf;
> @@ -111,17 +106,12 @@ struct sk_chain_data {
>   struct sk_chain sk_rx_chain[SK_RX_RING_CNT];
>   struct sk_txmap_entry *sk_tx_map[SK_TX_RING_CNT];
>   bus_dmamap_t sk_rx_map[SK_RX_RING_CNT];
> - bus_dmamap_t sk_rx_jumbo_map;
>   int sk_tx_prod;
>   int sk_tx_cons;
>   int sk_tx_cnt;
>   int sk_rx_prod;
>   int sk_rx_cons;
> - int sk_rx_cnt;
> - /* Stick the jumbo mem management stuff here too. */
> - caddr_t sk_jslots[SK_JSLOTS];
> - void *sk_jumbo_buf;
> -
> + struct if_rxring sk_rx_ring;
>   };
>
>   struct sk_ring_data {
> @@ -223,8 +213,6 @@ struct sk_if_softc {
>   int sk_ring_nseg;
>   struct sk_softc *sk_softc; /* parent controller */
>   int sk_tx_bmu; /* TX BMU register */
> - LIST_HEAD(__sk_jfreehead, sk_jpool_entry) sk_jfree_listhead;
> - LIST_HEAD(__sk_jinusehead, sk_jpool_entry) sk_jinuse_listhead;
>   SIMPLEQ_HEAD(__sk_txmaphead, sk_txmap_entry) sk_txmap_head;
>   };
>
>
>


--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
Cool, thanks.
On 19/08/2014 8:42 am, "Brad Smith" <[hidden email]> wrote:

> On 18/08/14 6:24 PM, David Gwynne wrote:
>
>> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>>
>>> i think i'll try to find the sk at work and wire it up. its just
>>> annoying cos im pretty sure its sr optics with sc connectors.
>>>
>>> thanks for testing.
>>>
>>
>> how's this one?
>>
>
> I'll look into testing it later on tonight or tomorrow.
>
>  Index: if_sk.c
>> ===================================================================
>> RCS file: /cvs/src/sys/dev/pci/if_sk.c,v
>> retrieving revision 1.170
>> diff -u -p -r1.170 if_sk.c
>> --- if_sk.c     22 Jul 2014 13:12:11 -0000      1.170
>> +++ if_sk.c     18 Aug 2014 21:57:14 -0000
>> @@ -153,12 +153,10 @@ void sk_watchdog(struct ifnet *);
>>   int sk_ifmedia_upd(struct ifnet *);
>>   void sk_ifmedia_sts(struct ifnet *, struct ifmediareq *);
>>   void skc_reset(struct sk_softc *);
>> -int sk_newbuf(struct sk_if_softc *, int, struct mbuf *, bus_dmamap_t);
>> -int sk_alloc_jumbo_mem(struct sk_if_softc *);
>> -void *sk_jalloc(struct sk_if_softc *);
>> -void sk_jfree(caddr_t, u_int, void *);
>> +int sk_newbuf(struct sk_if_softc *);
>>   int sk_reset(struct sk_if_softc *);
>>   int sk_init_rx_ring(struct sk_if_softc *);
>> +void sk_fill_rx_ring(struct sk_if_softc *);
>>   int sk_init_tx_ring(struct sk_if_softc *);
>>
>>   int sk_xmac_miibus_readreg(struct device *, int, int);
>> @@ -547,21 +545,29 @@ sk_init_rx_ring(struct sk_if_softc *sc_i
>>                 rd->sk_rx_ring[i].sk_next =
>> htole32(SK_RX_RING_ADDR(sc_if, nexti));
>>         }
>>
>> -       for (i = 0; i < SK_RX_RING_CNT; i++) {
>> -               if (sk_newbuf(sc_if, i, NULL,
>> -                   sc_if->sk_cdata.sk_rx_jumbo_map) == ENOBUFS) {
>> -                       printf("%s: failed alloc of %dth mbuf\n",
>> -                           sc_if->sk_dev.dv_xname, i);
>> -                       return (ENOBUFS);
>> -               }
>> -       }
>> -
>>         sc_if->sk_cdata.sk_rx_prod = 0;
>>         sc_if->sk_cdata.sk_rx_cons = 0;
>>
>> +       if_rxr_init(&sc_if->sk_cdata.sk_rx_ring, 2, SK_RX_RING_CNT);
>> +
>> +       sk_fill_rx_ring(sc_if);
>> +
>>         return (0);
>>   }
>>
>> +void
>> +sk_fill_rx_ring(struct sk_if_softc *sc_if)
>> +{
>> +       struct if_rxring *rxr = &sc_if->sk_cdata.sk_rx_ring;
>> +       u_int slots;
>> +
>> +       for (slots = if_rxr_get(rxr, SK_RX_RING_CNT); slots > 0; slots--)
>> {
>> +               if (sk_newbuf(sc_if) == ENOBUFS)
>> +                       break;
>> +       }
>> +       if_rxr_put(rxr, slots);
>> +}
>> +
>>   int
>>   sk_init_tx_ring(struct sk_if_softc *sc_if)
>>   {
>> @@ -609,199 +615,49 @@ sk_init_tx_ring(struct sk_if_softc *sc_i
>>   }
>>
>>   int
>> -sk_newbuf(struct sk_if_softc *sc_if, int i, struct mbuf *m,
>> -         bus_dmamap_t dmamap)
>> +sk_newbuf(struct sk_if_softc *sc_if)
>>   {
>> -       struct mbuf             *m_new = NULL;
>> +       struct mbuf             *m;
>>         struct sk_chain         *c;
>>         struct sk_rx_desc       *r;
>> +       bus_dmamap_t            dmamap;
>> +       u_int                   prod;
>> +       int                     error;
>>
>> -       if (m == NULL) {
>> -               caddr_t buf = NULL;
>> -
>> -               MGETHDR(m_new, M_DONTWAIT, MT_DATA);
>> -               if (m_new == NULL)
>> -                       return (ENOBUFS);
>> -
>> -               /* Allocate the jumbo buffer */
>> -               buf = sk_jalloc(sc_if);
>> -               if (buf == NULL) {
>> -                       m_freem(m_new);
>> -                       DPRINTFN(1, ("%s jumbo allocation failed --
>> packet "
>> -                           "dropped!\n", sc_if->arpcom.ac_if.if_xname));
>> -                       return (ENOBUFS);
>> -               }
>> -
>> -               /* Attach the buffer to the mbuf */
>> -               m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
>> -               MEXTADD(m_new, buf, SK_JLEN, 0, sk_jfree, sc_if);
>> -       } else {
>> -               /*
>> -                * We're re-using a previously allocated mbuf;
>> -                * be sure to re-init pointers and lengths to
>> -                * default values.
>> -                */
>> -               m_new = m;
>> -               m_new->m_len = m_new->m_pkthdr.len = SK_JLEN;
>> -               m_new->m_data = m_new->m_ext.ext_buf;
>> -       }
>> -       m_adj(m_new, ETHER_ALIGN);
>> -
>> -       c = &sc_if->sk_cdata.sk_rx_chain[i];
>> -       r = c->sk_desc;
>> -       c->sk_mbuf = m_new;
>> -       r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr +
>> -           (((vaddr_t)m_new->m_data
>> -             - (vaddr_t)sc_if->sk_cdata.sk_jumbo_buf)));
>> -       r->sk_ctl = htole32(SK_JLEN | SK_RXSTAT);
>> -
>> -       SK_CDRXSYNC(sc_if, i, BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
>> +       m = MCLGETI(NULL, M_DONTWAIT, NULL, SK_JLEN);
>> +       if (m == NULL)
>> +               return (ENOBUFS);
>>
>> -       return (0);
>> -}
>> +        m->m_len = m->m_pkthdr.len = SK_JLEN;
>> +       m_adj(m, ETHER_ALIGN);
>>
>> -/*
>> - * Memory management for jumbo frames.
>> - */
>> +       prod = sc_if->sk_cdata.sk_rx_prod;
>> +       dmamap = sc_if->sk_cdata.sk_rx_map[prod];
>>
>> -int
>> -sk_alloc_jumbo_mem(struct sk_if_softc *sc_if)
>> -{
>> -       struct sk_softc         *sc = sc_if->sk_softc;
>> -       caddr_t                 ptr, kva;
>> -       bus_dma_segment_t       seg;
>> -       int             i, rseg, state, error;
>> -       struct sk_jpool_entry   *entry;
>> -
>> -       state = error = 0;
>> -
>> -       /* Grab a big chunk o' storage. */
>> -       if (bus_dmamem_alloc(sc->sc_dmatag, SK_JMEM, PAGE_SIZE, 0,
>> -                            &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
>> -               printf(": can't alloc rx buffers");
>> +       error = bus_dmamap_load_mbuf(sc_if->sk_softc->sc_dmatag, dmamap,
>> m,
>> +           BUS_DMA_READ|BUS_DMA_NOWAIT);
>> +       if (error) {
>> +               m_freem(m);
>>                 return (ENOBUFS);
>>         }
>>
>> -       state = 1;
>> -       if (bus_dmamem_map(sc->sc_dmatag, &seg, rseg, SK_JMEM, &kva,
>> -                          BUS_DMA_NOWAIT)) {
>> -               printf(": can't map dma buffers (%d bytes)", SK_JMEM);
>> -               error = ENOBUFS;
>> -               goto out;
>> -       }
>> -
>> -       state = 2;
>> -       if (bus_dmamap_create(sc->sc_dmatag, SK_JMEM, 1, SK_JMEM, 0,
>> -           BUS_DMA_NOWAIT, &sc_if->sk_cdata.sk_rx_jumbo_map)) {
>> -               printf(": can't create dma map");
>> -               error = ENOBUFS;
>> -               goto out;
>> -       }
>> -
>> -       state = 3;
>> -       if (bus_dmamap_load(sc->sc_dmatag, sc_if->sk_cdata.sk_rx_jumbo_
>> map,
>> -                           kva, SK_JMEM, NULL, BUS_DMA_NOWAIT)) {
>> -               printf(": can't load dma map");
>> -               error = ENOBUFS;
>> -               goto out;
>> -       }
>> -
>> -       state = 4;
>> -       sc_if->sk_cdata.sk_jumbo_buf = (caddr_t)kva;
>> -       DPRINTFN(1,("sk_jumbo_buf = 0x%08X\n",
>> sc_if->sk_cdata.sk_jumbo_buf));
>> +       bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
>> +           dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
>>
>> -       LIST_INIT(&sc_if->sk_jfree_listhead);
>> -       LIST_INIT(&sc_if->sk_jinuse_listhead);
>> +       c = &sc_if->sk_cdata.sk_rx_chain[prod];
>> +       c->sk_mbuf = m;
>>
>> -       /*
>> -        * Now divide it up into 9K pieces and save the addresses
>> -        * in an array.
>> -        */
>> -       ptr = sc_if->sk_cdata.sk_jumbo_buf;
>> -       for (i = 0; i < SK_JSLOTS; i++) {
>> -               sc_if->sk_cdata.sk_jslots[i] = ptr;
>> -               ptr += SK_JLEN;
>> -               entry = malloc(sizeof(struct sk_jpool_entry),
>> -                   M_DEVBUF, M_NOWAIT);
>> -               if (entry == NULL) {
>> -                       sc_if->sk_cdata.sk_jumbo_buf = NULL;
>> -                       printf(": no memory for jumbo buffer queue!");
>> -                       error = ENOBUFS;
>> -                       goto out;
>> -               }
>> -               entry->slot = i;
>> -               LIST_INSERT_HEAD(&sc_if->sk_jfree_listhead,
>> -                                entry, jpool_entries);
>> -       }
>> -out:
>> -       if (error != 0) {
>> -               switch (state) {
>> -               case 4:
>> -                       bus_dmamap_unload(sc->sc_dmatag,
>> -                           sc_if->sk_cdata.sk_rx_jumbo_map);
>> -               case 3:
>> -                       bus_dmamap_destroy(sc->sc_dmatag,
>> -                           sc_if->sk_cdata.sk_rx_jumbo_map);
>> -               case 2:
>> -                       bus_dmamem_unmap(sc->sc_dmatag, kva, SK_JMEM);
>> -               case 1:
>> -                       bus_dmamem_free(sc->sc_dmatag, &seg, rseg);
>> -                       break;
>> -               default:
>> -                       break;
>> -               }
>> -       }
>> -
>> -       return (error);
>> -}
>> -
>> -/*
>> - * Allocate a jumbo buffer.
>> - */
>> -void *
>> -sk_jalloc(struct sk_if_softc *sc_if)
>> -{
>> -       struct sk_jpool_entry   *entry;
>> -
>> -       entry = LIST_FIRST(&sc_if->sk_jfree_listhead);
>> -
>> -       if (entry == NULL)
>> -               return (NULL);
>> -
>> -       LIST_REMOVE(entry, jpool_entries);
>> -       LIST_INSERT_HEAD(&sc_if->sk_jinuse_listhead, entry,
>> jpool_entries);
>> -       return (sc_if->sk_cdata.sk_jslots[entry->slot]);
>> -}
>> -
>> -/*
>> - * Release a jumbo buffer.
>> - */
>> -void
>> -sk_jfree(caddr_t buf, u_int size, void *arg)
>> -{
>> -       struct sk_jpool_entry *entry;
>> -       struct sk_if_softc *sc;
>> -       int i;
>> +       r = c->sk_desc;
>> +       r->sk_data_lo = htole32(dmamap->dm_segs[0].ds_addr);
>> +       r->sk_data_hi = htole32(((u_int64_t)dmamap->dm_segs[0].ds_addr)
>> >> 32);
>> +       r->sk_ctl = htole32(dmamap->dm_segs[0].ds_len | SK_RXSTAT);
>>
>> -       /* Extract the softc struct pointer. */
>> -       sc = (struct sk_if_softc *)arg;
>> +       SK_CDRXSYNC(sc_if, prod, BUS_DMASYNC_PREWRITE|BUS_
>> DMASYNC_PREREAD);
>>
>> -       if (sc == NULL)
>> -               panic("sk_jfree: can't find softc pointer!");
>> +       SK_INC(prod, SK_RX_RING_CNT);
>> +       sc_if->sk_cdata.sk_rx_prod = prod;
>>
>> -       /* calculate the slot this buffer belongs to */
>> -       i = ((vaddr_t)buf
>> -            - (vaddr_t)sc->sk_cdata.sk_jumbo_buf) / SK_JLEN;
>> -
>> -       if ((i < 0) || (i >= SK_JSLOTS))
>> -               panic("sk_jfree: asked to free buffer that we don't
>> manage!");
>> -
>> -       entry = LIST_FIRST(&sc->sk_jinuse_listhead);
>> -       if (entry == NULL)
>> -               panic("sk_jfree: buffer not in use!");
>> -       entry->slot = i;
>> -       LIST_REMOVE(entry, jpool_entries);
>> -       LIST_INSERT_HEAD(&sc->sk_jfree_listhead, entry, jpool_entries);
>> +       return (0);
>>   }
>>
>>   /*
>> @@ -869,6 +725,12 @@ sk_ioctl(struct ifnet *ifp, u_long comma
>>                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
>>                 break;
>>
>> +       case SIOCGIFRXR:
>> +               error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
>> +                   NULL, SK_JLEN, &sc_if->sk_cdata.sk_rx_ring);
>> +
>> +               break;
>> +
>>         default:
>>                 error = ether_ioctl(ifp, &sc_if->arpcom, command, data);
>>         }
>> @@ -997,7 +859,7 @@ sk_attach(struct device *parent, struct
>>         struct skc_attach_args *sa = aux;
>>         struct ifnet *ifp;
>>         caddr_t kva;
>> -       int i;
>> +       int i, error;
>>
>>         sc_if->sk_port = sa->skc_port;
>>         sc_if->sk_softc = sc;
>> @@ -1124,10 +986,14 @@ sk_attach(struct device *parent, struct
>>         }
>>           sc_if->sk_rdata = (struct sk_ring_data *)kva;
>>
>> -       /* Try to allocate memory for jumbo buffers. */
>> -       if (sk_alloc_jumbo_mem(sc_if)) {
>> -               printf(": jumbo buffer allocation failed\n");
>> -               goto fail_3;
>> +       for (i = 0; i < SK_RX_RING_CNT; i++) {
>> +               error = bus_dmamap_create(sc->sc_dmatag, SK_JLEN, 1,
>> +                   SK_JLEN, 0, 0, &sc_if->sk_cdata.sk_rx_map[i]);
>> +               if (error != 0) {
>> +                       printf(": unable to create rx DMA map %d, "
>> +                           "error = %d\n", i, error);
>> +                       goto fail_4;
>> +               }
>>         }
>>
>>         ifp = &sc_if->arpcom.ac_if;
>> @@ -1193,12 +1059,18 @@ sk_attach(struct device *parent, struct
>>
>>         DPRINTFN(2, ("sk_attach: end\n"));
>>         return;
>> +fail_4:
>> +       for (i = 0; i < SK_RX_RING_CNT; i++) {
>> +               if (sc_if->sk_cdata.sk_rx_map[i] == NULL)
>> +                       continue;
>>
>> -fail_2:
>> +               bus_dmamap_destroy(sc->sc_dmatag,
>> sc_if->sk_cdata.sk_rx_map[i]);
>> +       }
>> +fail_3:
>>         bus_dmamem_unmap(sc->sc_dmatag, kva, sizeof(struct
>> sk_ring_data));
>> -fail_1:
>> +fail_2:
>>         bus_dmamem_free(sc->sc_dmatag, &sc_if->sk_ring_seg,
>> sc_if->sk_ring_nseg);
>> -fail_3:
>> +fail_1:
>>         bus_dmamap_destroy(sc->sc_dmatag, sc_if->sk_ring_map);
>>   fail:
>>         sc->sk_if[sa->skc_port] = NULL;
>> @@ -1718,45 +1590,44 @@ sk_rxeof(struct sk_if_softc *sc_if)
>>   {
>>         struct sk_softc         *sc = sc_if->sk_softc;
>>         struct ifnet            *ifp = &sc_if->arpcom.ac_if;
>> +       struct if_rxring        *rxr = &sc_if->sk_cdata.sk_rx_ring;
>>         struct mbuf             *m;
>>         struct sk_chain         *cur_rx;
>>         struct sk_rx_desc       *cur_desc;
>> -       int                     i, cur, total_len = 0;
>> +       int                     cur, total_len = 0;
>>         u_int32_t               rxstat, sk_ctl;
>>         bus_dmamap_t            dmamap;
>>
>>         DPRINTFN(2, ("sk_rxeof\n"));
>>
>> -       i = sc_if->sk_cdata.sk_rx_prod;
>> -
>> -       for (;;) {
>> -               cur = i;
>> -
>> +       cur = sc_if->sk_cdata.sk_rx_cons;
>> +       while (if_rxr_inuse(rxr) > 0) {
>>                 /* Sync the descriptor */
>>                 SK_CDRXSYNC(sc_if, cur,
>>                     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
>>
>> -               sk_ctl = letoh32(sc_if->sk_rdata->sk_rx_ring[i].sk_ctl);
>> -               if ((sk_ctl & SK_RXCTL_OWN) != 0) {
>> -                       /* Invalidate the descriptor -- it's not ready
>> yet */
>> -                       SK_CDRXSYNC(sc_if, cur, BUS_DMASYNC_PREREAD);
>> -                       sc_if->sk_cdata.sk_rx_prod = i;
>> +               cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
>> +               if (cur_rx->sk_mbuf == NULL)
>> +                       break;
>> +
>> +               sk_ctl = letoh32(sc_if->sk_rdata->sk_
>> rx_ring[cur].sk_ctl);
>> +               if ((sk_ctl & SK_RXCTL_OWN) != 0)
>>                         break;
>> -               }
>>
>> -               cur_rx = &sc_if->sk_cdata.sk_rx_chain[cur];
>>                 cur_desc = &sc_if->sk_rdata->sk_rx_ring[cur];
>> -               dmamap = sc_if->sk_cdata.sk_rx_jumbo_map;
>> +               dmamap = sc_if->sk_cdata.sk_rx_map[cur];
>>
>>                 bus_dmamap_sync(sc_if->sk_softc->sc_dmatag, dmamap, 0,
>>                     dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
>> +               bus_dmamap_unload(sc_if->sk_softc->sc_dmatag, dmamap);
>>
>> -               rxstat = letoh32(cur_desc->sk_xmac_rxstat);
>>                 m = cur_rx->sk_mbuf;
>>                 cur_rx->sk_mbuf = NULL;
>> -               total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
>> +               if_rxr_put(rxr, 1);
>> +               SK_INC(cur, SK_RX_RING_CNT);
>>
>> -               SK_INC(i, SK_RX_RING_CNT);
>> +               total_len = SK_RXBYTES(letoh32(cur_desc->sk_ctl));
>> +               rxstat = letoh32(cur_desc->sk_xmac_rxstat);
>>
>>                 if ((sk_ctl & (SK_RXCTL_STATUS_VALID | SK_RXCTL_FIRSTFRAG
>> |
>>                     SK_RXCTL_LASTFRAG)) != (SK_RXCTL_STATUS_VALID |
>> @@ -1765,31 +1636,12 @@ sk_rxeof(struct sk_if_softc *sc_if)
>>                     total_len > SK_JUMBO_FRAMELEN ||
>>                     sk_rxvalid(sc, rxstat, total_len) == 0) {
>>                         ifp->if_ierrors++;
>> -                       sk_newbuf(sc_if, cur, m, dmamap);
>> +                       m_freem(m);
>>                         continue;
>>                 }
>>
>> -               /*
>> -                * Try to allocate a new jumbo buffer. If that
>> -                * fails, copy the packet to mbufs and put the
>> -                * jumbo buffer back in the ring so it can be
>> -                * re-used. If allocating mbufs fails, then we
>> -                * have to drop the packet.
>> -                */
>> -               if (sk_newbuf(sc_if, cur, NULL, dmamap) == ENOBUFS) {
>> -                       struct mbuf             *m0;
>> -                       m0 = m_devget(mtod(m, char *), total_len,
>> ETHER_ALIGN,
>> -                           ifp);
>> -                       sk_newbuf(sc_if, cur, m, dmamap);
>> -                       if (m0 == NULL) {
>> -                               ifp->if_ierrors++;
>> -                               continue;
>> -                       }
>> -                       m = m0;
>> -               } else {
>> -                       m->m_pkthdr.rcvif = ifp;
>> -                       m->m_pkthdr.len = m->m_len = total_len;
>> -               }
>> +               m->m_pkthdr.rcvif = ifp;
>> +               m->m_pkthdr.len = m->m_len = total_len;
>>
>>                 ifp->if_ipackets++;
>>
>> @@ -1801,6 +1653,9 @@ sk_rxeof(struct sk_if_softc *sc_if)
>>                 /* pass it on. */
>>                 ether_input_mbuf(ifp, m);
>>         }
>> +       sc_if->sk_cdata.sk_rx_cons = cur;
>> +
>> +       sk_fill_rx_ring(sc_if);
>>   }
>>
>>   void
>> @@ -2570,6 +2425,7 @@ sk_stop(struct sk_if_softc *sc_if, int s
>>   {
>>         struct sk_softc         *sc = sc_if->sk_softc;
>>         struct ifnet            *ifp = &sc_if->arpcom.ac_if;
>> +       bus_dmamap_t            dmamap;
>>         struct sk_txmap_entry   *dma;
>>         int                     i;
>>         u_int32_t               val;
>> @@ -2660,6 +2516,10 @@ sk_stop(struct sk_if_softc *sc_if, int s
>>         /* Free RX and TX mbufs still in the queues. */
>>         for (i = 0; i < SK_RX_RING_CNT; i++) {
>>                 if (sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf != NULL) {
>> +                       dmamap = sc_if->sk_cdata.sk_rx_map[i];
>> +                       bus_dmamap_sync(sc_if->sk_softc->sc_dmatag,
>> dmamap, 0,
>> +                           dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD);
>> +                       bus_dmamap_unload(sc_if->sk_softc->sc_dmatag,
>> dmamap);
>>                         m_freem(sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf);
>>                         sc_if->sk_cdata.sk_rx_chain[i].sk_mbuf = NULL;
>>                 }
>> Index: if_skvar.h
>> ===================================================================
>> RCS file: /cvs/src/sys/dev/pci/if_skvar.h,v
>> retrieving revision 1.8
>> diff -u -p -r1.8 if_skvar.h
>> --- if_skvar.h  7 Mar 2013 06:13:31 -0000       1.8
>> +++ if_skvar.h  18 Aug 2014 21:57:14 -0000
>> @@ -81,11 +81,6 @@
>>   #ifndef _DEV_PCI_IF_SKVAR_H_
>>   #define _DEV_PCI_IF_SKVAR_H_
>>
>> -struct sk_jpool_entry {
>> -       int                             slot;
>> -       LIST_ENTRY(sk_jpool_entry)      jpool_entries;
>> -};
>> -
>>   struct sk_chain {
>>         void                    *sk_desc;
>>         struct mbuf             *sk_mbuf;
>> @@ -111,17 +106,12 @@ struct sk_chain_data {
>>         struct sk_chain         sk_rx_chain[SK_RX_RING_CNT];
>>         struct sk_txmap_entry   *sk_tx_map[SK_TX_RING_CNT];
>>         bus_dmamap_t            sk_rx_map[SK_RX_RING_CNT];
>> -       bus_dmamap_t            sk_rx_jumbo_map;
>>         int                     sk_tx_prod;
>>         int                     sk_tx_cons;
>>         int                     sk_tx_cnt;
>>         int                     sk_rx_prod;
>>         int                     sk_rx_cons;
>> -       int                     sk_rx_cnt;
>> -       /* Stick the jumbo mem management stuff here too. */
>> -       caddr_t                 sk_jslots[SK_JSLOTS];
>> -       void                    *sk_jumbo_buf;
>> -
>> +       struct if_rxring        sk_rx_ring;
>>   };
>>
>>   struct sk_ring_data {
>> @@ -223,8 +213,6 @@ struct sk_if_softc {
>>         int                     sk_ring_nseg;
>>         struct sk_softc         *sk_softc;      /* parent controller */
>>         int                     sk_tx_bmu;      /* TX BMU register */
>> -       LIST_HEAD(__sk_jfreehead, sk_jpool_entry)       sk_jfree_listhead;
>> -       LIST_HEAD(__sk_jinusehead, sk_jpool_entry)
>> sk_jinuse_listhead;
>>         SIMPLEQ_HEAD(__sk_txmaphead, sk_txmap_entry)    sk_txmap_head;
>>   };
>>
>>
>>
>>
>
> --
> This message has been scanned for viruses and
> dangerous content by MailScanner, and is
> believed to be clean.
>
>
Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Higgs
In reply to this post by David Gwynne-5
On Mon, Aug 18, 2014 at 6:24 PM, David Gwynne <[hidden email]> wrote:
> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>> i think i'll try to find the sk at work and wire it up. its just annoying cos im pretty sure its sr optics with sc connectors.
>>
>> thanks for testing.
>
> how's this one?
>

Are either of you (or anyone else) interested in one or both of my
twisted-pair sk(4) cards?  I've plugged maybe once or twice in the
last few years and maybe they can be put to better use elsewhere.  I
never saw them in want.html and never offered because I figured they
fell in the "very common PCI" category, but contact me off-list if
this isn't the case to help me figure out where to ship them to.

Full dmesg: http://marc.info/?l=openbsd-tech&m=139062896512049&w=2

skc0 at pci0 dev 16 function 0 "3Com 3c940" rev 0x10, Yukon (0x1): irq 9
sk0 at skc0 port A: address 00:0a:5e:5c:50:41
eephy0 at sk0 phy 0: 88E1011 Gigabit PHY, rev. 3
skc1 at pci0 dev 17 function 0 "3Com 3c940" rev 0x10, Yukon (0x1): irq 10
sk1 at skc1 port A: address 00:0a:5e:65:ee:47
eephy1 at sk1 phy 0: 88E1011 Gigabit PHY, rev. 3

Thanks.

--david

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
someone already sent me one to test with (hence the diff) at the same time as one of the guys at work found an old fibre one here and wired it up for me.

i think i have enough of them now, but thank you for the offer.

cheers,
dlg

On 19 Aug 2014, at 11:17, David Higgs <[hidden email]> wrote:

> On Mon, Aug 18, 2014 at 6:24 PM, David Gwynne <[hidden email]> wrote:
>> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>>> i think i'll try to find the sk at work and wire it up. its just annoying cos im pretty sure its sr optics with sc connectors.
>>>
>>> thanks for testing.
>>
>> how's this one?
>>
>
> Are either of you (or anyone else) interested in one or both of my
> twisted-pair sk(4) cards?  I've plugged maybe once or twice in the
> last few years and maybe they can be put to better use elsewhere.  I
> never saw them in want.html and never offered because I figured they
> fell in the "very common PCI" category, but contact me off-list if
> this isn't the case to help me figure out where to ship them to.
>
> Full dmesg: http://marc.info/?l=openbsd-tech&m=139062896512049&w=2
>
> skc0 at pci0 dev 16 function 0 "3Com 3c940" rev 0x10, Yukon (0x1): irq 9
> sk0 at skc0 port A: address 00:0a:5e:5c:50:41
> eephy0 at sk0 phy 0: 88E1011 Gigabit PHY, rev. 3
> skc1 at pci0 dev 17 function 0 "3Com 3c940" rev 0x10, Yukon (0x1): irq 10
> sk1 at skc1 port A: address 00:0a:5e:65:ee:47
> eephy1 at sk1 phy 0: 88E1011 Gigabit PHY, rev. 3
>
> Thanks.
>
> --david


Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Brad Smith-14
In reply to this post by David Gwynne-5
On 18/08/14 6:24 PM, David Gwynne wrote:
> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>> i think i'll try to find the sk at work and wire it up. its just annoying cos im pretty sure its sr optics with sc connectors.
>>
>> thanks for testing.
>
> how's this one?

Only running regular MTU traffic so far but the NIC at least works and
beating on it I haven't seen any noticeable issues so far. I'll see if
I can do some test with Jumbos a little later on today.

sk0                          9020    19     2   256    19

sk0     1500  <Link>      00:00:5a:98:b9:c0 29810991     0 23786214
0     0

skc0 at pci0 dev 5 function 0 "Schneider & Koch SK-98xx" rev 0x13,
GEnesis (0x0): ivec 0x784
sk0 at skc0 port A: address 00:00:5a:98:b9:c0
brgphy0 at sk0 phy 1: BCM5400 1000baseT PHY, rev. 7

Some minor cosmetic issues I noticed while diff'ing the two revisions
of the diff...

m->m_len = m->m_pkthdr.len = SK_JLEN;

has spaces instead of a tab and the whitespace after if_rxr_ioctl().

--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Stuart Henderson-6
From what I remember from last attempt to convert sk(4) to MCLGETI,
there were problems which only showed up under load (possibly also involved
NFS, I don't remember for sure) - I probably used netrate with something like
"netblast 11.22.33.44 12345 1 30" to generate a bunch of packets over it
quickly, and tcpbench to generate high bandwidth use.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Brad Smith-14
On 19/08/14 2:43 PM, Stuart Henderson wrote:
>  From what I remember from last attempt to convert sk(4) to MCLGETI,
> there were problems which only showed up under load (possibly also involved
> NFS, I don't remember for sure) - I probably used netrate with something like
> "netblast 11.22.33.44 12345 1 30" to generate a bunch of packets over it
> quickly, and tcpbench to generate high bandwidth use.

The first diff he sent out didn't work at all.


--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
I think sthen is referring to the original diff Kettenis had in the tree a
few years ago.
On 20/08/2014 4:44 am, "Brad Smith" <[hidden email]> wrote:

> On 19/08/14 2:43 PM, Stuart Henderson wrote:
>
>>  From what I remember from last attempt to convert sk(4) to MCLGETI,
>> there were problems which only showed up under load (possibly also
>> involved
>> NFS, I don't remember for sure) - I probably used netrate with something
>> like
>> "netblast 11.22.33.44 12345 1 30" to generate a bunch of packets over it
>> quickly, and tcpbench to generate high bandwidth use.
>>
>
> The first diff he sent out didn't work at all.
>
>
> --
> This message has been scanned for viruses and
> dangerous content by MailScanner, and is
> believed to be clean.
>
>
Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Stuart Henderson-6
On 2014/08/20 07:06, David Gwynne wrote:
> I think sthen is referring to the original diff Kettenis had in the tree a
> few years ago.

Yes - I just wanted to give ideas of things that might be worth testing
that have been known to cause problems on those NICs in the past


> On 20/08/2014 4:44 am, "Brad Smith" <[hidden email]> wrote:
>
> > On 19/08/14 2:43 PM, Stuart Henderson wrote:
> >
> >>  From what I remember from last attempt to convert sk(4) to MCLGETI,
> >> there were problems which only showed up under load (possibly also
> >> involved
> >> NFS, I don't remember for sure) - I probably used netrate with something
> >> like
> >> "netblast 11.22.33.44 12345 1 30" to generate a bunch of packets over it
> >> quickly, and tcpbench to generate high bandwidth use.
> >>
> >
> > The first diff he sent out didn't work at all.
> >
> >
> > --
> > This message has been scanned for viruses and
> > dangerous content by MailScanner, and is
> > believed to be clean.
> >
> >

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

Brad Smith-14
In reply to this post by Brad Smith-14
On 19/08/14 2:19 PM, Brad Smith wrote:

> On 18/08/14 6:24 PM, David Gwynne wrote:
>> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>>> i think i'll try to find the sk at work and wire it up. its just
>>> annoying cos im pretty sure its sr optics with sc connectors.
>>>
>>> thanks for testing.
>>
>> how's this one?
>
> Only running regular MTU traffic so far but the NIC at least works and
> beating on it I haven't seen any noticeable issues so far. I'll see if
> I can do some test with Jumbos a little later on today.
>
> sk0                          9020    19     2   256    19
>
> sk0     1500  <Link>      00:00:5a:98:b9:c0 29810991     0 23786214 0     0
>
> skc0 at pci0 dev 5 function 0 "Schneider & Koch SK-98xx" rev 0x13,
> GEnesis (0x0): ivec 0x784
> sk0 at skc0 port A: address 00:00:5a:98:b9:c0
> brgphy0 at sk0 phy 1: BCM5400 1000baseT PHY, rev. 7
>
> Some minor cosmetic issues I noticed while diff'ing the two revisions
> of the diff...
>
> m->m_len = m->m_pkthdr.len = SK_JLEN;
>
> has spaces instead of a tab and the whitespace after if_rxr_ioctl().

Running either netrate or tcpbench targetted at the test system and
its still working Ok. No packet loss.

--
This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.

Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
In reply to this post by Stuart Henderson-6

On 20 Aug 2014, at 7:12 am, Stuart Henderson <[hidden email]> wrote:

> On 2014/08/20 07:06, David Gwynne wrote:
>> I think sthen is referring to the original diff Kettenis had in the tree a
>> few years ago.
>
> Yes - I just wanted to give ideas of things that might be worth testing
> that have been known to cause problems on those NICs in the past

indeed.

i have tested high packet rates and vlans. ill kick the tyres on jumbos again and try a cvs co and kernel build on nfs if i get a spare moment at work.

dlg

>
>
>> On 20/08/2014 4:44 am, "Brad Smith" <[hidden email]> wrote:
>>
>>> On 19/08/14 2:43 PM, Stuart Henderson wrote:
>>>
>>>> From what I remember from last attempt to convert sk(4) to MCLGETI,
>>>> there were problems which only showed up under load (possibly also
>>>> involved
>>>> NFS, I don't remember for sure) - I probably used netrate with something
>>>> like
>>>> "netblast 11.22.33.44 12345 1 30" to generate a bunch of packets over it
>>>> quickly, and tcpbench to generate high bandwidth use.
>>>>
>>>
>>> The first diff he sent out didn't work at all.
>>>
>>>
>>> --
>>> This message has been scanned for viruses and
>>> dangerous content by MailScanner, and is
>>> believed to be clean.
>>>
>>>


Reply | Threaded
Open this post in threaded view
|

Re: sk(4): jumbo mbufs and rxring accounting

David Gwynne-5
In reply to this post by Brad Smith-14
jumbos seem find. nfs seems fine.

im going to put this and the other jalloc driver changes in to see what happens.

dlg

On 20 Aug 2014, at 7:41, Brad Smith <[hidden email]> wrote:

> On 19/08/14 2:19 PM, Brad Smith wrote:
>> On 18/08/14 6:24 PM, David Gwynne wrote:
>>> On Sun, Jul 13, 2014 at 02:01:15PM +1000, David Gwynne wrote:
>>>> i think i'll try to find the sk at work and wire it up. its just
>>>> annoying cos im pretty sure its sr optics with sc connectors.
>>>>
>>>> thanks for testing.
>>>
>>> how's this one?
>>
>> Only running regular MTU traffic so far but the NIC at least works and
>> beating on it I haven't seen any noticeable issues so far. I'll see if
>> I can do some test with Jumbos a little later on today.
>>
>> sk0                          9020    19     2   256    19
>>
>> sk0     1500  <Link>      00:00:5a:98:b9:c0 29810991     0 23786214 0     0
>>
>> skc0 at pci0 dev 5 function 0 "Schneider & Koch SK-98xx" rev 0x13,
>> GEnesis (0x0): ivec 0x784
>> sk0 at skc0 port A: address 00:00:5a:98:b9:c0
>> brgphy0 at sk0 phy 1: BCM5400 1000baseT PHY, rev. 7
>>
>> Some minor cosmetic issues I noticed while diff'ing the two revisions
>> of the diff...
>>
>> m->m_len = m->m_pkthdr.len = SK_JLEN;
>>
>> has spaces instead of a tab and the whitespace after if_rxr_ioctl().
>
> Running either netrate or tcpbench targetted at the test system and
> its still working Ok. No packet loss.
>
> --
> This message has been scanned for viruses and
> dangerous content by MailScanner, and is
> believed to be clean.