AnsweredAssumed Answered

Intermittent drop of Ethernet transmit packets

Question asked by Green.Element on Jan 28, 2016
Latest reply on Jan 29, 2016 by Green.Element
I've designed a STM32F107 based board with a Microchip LAN8720A PHY.  The code I am using has been pieced together from several sources and is using lwIP 1.4.1 and FreeRTOS.

I have a working HTTP and telnet server.  However, occasionally transmit packets are being dropped (verified through code logging and sniffing of the network traffic), resulting in about a second of delay when the lwIP stack re-transmits the last packet which was lost.

I'm using chained DMA Ethernet descriptors, with 2 receive descriptors and 1 transmit descriptor.  I have the interrupt service routine configured with only Receive interrupts which synchronizes with a task through a semaphore for processing the incoming data.  The transmit side checks the OWN bit of the transmit descriptor and continually polls it with a slight delay between retries (stalls the calling code until transmit is possible).

I'm fairly certain the issue only occurs if the transmitted data spans several packets.   The first 2 packets get sent back to back, prior to receiving an ACK to the first one.  The second packet is the one that gets dropped intermittently.  I've been working on this issue for almost a week and have been unable to find a solution.  My first attempts to track it down involved a log routine to the serial port.  When placing a call to the logging functions in the transmit routine, the problem went away, due to the added delay I presume.  I then created a logging function which is much faster which does not mask the problem and I am using it to look at the sequence of events.

Does anyone have any tips or thoughts of why this might be happening?  It seems like some sort of race condition is occurring with the transmit descriptor OWN bit, where it is not really ready yet or something, or perhaps the DMA transfer is failing (though I have been unable to trap it).  I have seen reference code floating around for FreeRTOS and this processor which has a reference to such a race condition and actually sends every packet twice.  I found this to be very strange and wondered if it was a workaround for some sort of errata or other issue.

Any help would be much appreciated.  I've pasted the relevant code below (sorry for the double spacing, something wrong with cut/paste to this form):

// Ethernet reset delay in ms (LAN8720 spec sheet says min of 25ms from 80% power supply voltage to reset deassert)
#define ETHERNET_RESET_DELAY_MS 40


#define netifMTU                        1500
#define ETHRX_TASK_STACK_SIZE           600
#define ETHRX_TASK_PRIORITY             3
#define IFNAME0 'd'
#define IFNAME1 'a'


#define NUM_RX_DESCRIPTORS      2   // Receive DMA chain descriptor count
#define NUM_TX_DESCRIPTORS      1   // Transmit DMA chain descriptor count


#define NUM_RX_BUFFERS          NUM_RX_DESCRIPTORS      // Number of receive DMA buffers total
#define NUM_TX_BUFFERS          NUM_TX_DESCRIPTORS      // Number of transmit DMA buffers total


#define MAX_PACKET_SIZE         1520    // Max ETH frame length


#define TX_QUEUE_LEN            8       // Max number of pbuf pointers to queue up for transmit, before blocking


#define  ETH_DMARxDesc_FrameLengthShift         16      // Number of bits to shift DMA status register for frame length value


/* Allocate the Rx and descriptors used by the DMA. */
static ETH_DMADESCTypeDef xRxDescriptors[NUM_RX_DESCRIPTORS] __attribute__((aligned(4)));
static ETH_DMADESCTypeDef xTxDescriptors[NUM_TX_DESCRIPTORS] __attribute__((aligned(4)));


/* Allocate separated buffers for receive and transmit data*/
static unsigned char rxMACBuffers[NUM_RX_BUFFERS][MAX_PACKET_SIZE] __attribute__((aligned(4)));
static unsigned char txMACBuffers[NUM_TX_BUFFERS][MAX_PACKET_SIZE] __attribute__((aligned(4)));


extern ETH_DMADESCTypeDef *DMATxDescToSet;
extern ETH_DMADESCTypeDef *DMARxDescToGet;


static SemaphoreHandle_t ETH_RX_Sem;    // The semaphore used by the ISR to wake the lwIP task
static xSemaphoreHandle ETH_TX_Mutex;   // Mutex for transmit DMA


#define netifINIT_WAIT  (100 / portTICK_RATE_MS)




/* Forward declarations. */
static err_t low_level_output (struct netif *netif, struct pbuf *p);
static struct pbuf *low_level_input (struct netif *netif);
void ethRxTask (void *param);
static void ethGpioInit (void);
static portBASE_TYPE ethMacInit (UINT8 *ucMACAddress);




/**
 * Should be called at the beginning of the program to set up the
 * network interface. It calls the function low_level_init() to do the
 * actual setup of the hardware.
 *
 * This function should be passed as a parameter to netif_add().
 *
 * @param netif the lwip network interface structure for this ethernetif
 * @return ERR_OK if the loopif is initialized
 *         ERR_MEM if private data couldn't be allocated
 *         any other err_t on error
 */
err_t
ethernetif_init (struct netif *netif)
{
  LWIP_ASSERT ("netif != NULL", (netif != NULL));


#if LWIP_NETIF_HOSTNAME
  /* Initialize interface hostname */
  netif->hostname = "lwip";
#endif /* LWIP_NETIF_HOSTNAME */


  // Initialize SNMP variables and link speed (last parameter in bits per second)
  NETIF_INIT_SNMP (netif, snmp_ifType_ethernet_csmacd, 100000000);


  netif->state = NULL;
  netif->name[0] = IFNAME0;
  netif->name[1] = IFNAME1;


  netif->output = etharp_output;
  netif->linkoutput = low_level_output;


  // Create the receive semaphore
  ETH_RX_Sem = xSemaphoreCreateBinary ();
  LWIP_ASSERT ("ETH_RX_Sem != NULL", ETH_RX_Sem != NULL);
  xSemaphoreTake (ETH_RX_Sem, 0);


  ETH_TX_Mutex = xSemaphoreCreateMutex ();


  netif->hwaddr_len = ETHARP_HWADDR_LEN;
  memcpy (netif->hwaddr, &sysConfig.macAddr, 6);


  netif->mtu = netifMTU;
  netif->flags |= NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP| NETIF_FLAG_LINK_UP;


  // Initialize the MAC
  while (ethMacInit (netif->hwaddr) != pdPASS)
    vTaskDelay (netifINIT_WAIT);


  // Create packet receive task
  sys_thread_new ("EthRx", ethRxTask, netif, ETHRX_TASK_STACK_SIZE, ETHRX_TASK_PRIORITY);


  return ERR_OK;
}


/**
 * Ethernet packet receive processing task.
 *
 * @param netif the lwip network interface structure for this ethernetif
 */
void
ethRxTask (void *param)
{
  struct netif *netif = param;
  struct pbuf *p;


  while (TRUE)
  {
    xSemaphoreTake (ETH_RX_Sem, portMAX_DELAY);         // Wait for packet or receive overflow error


    while (TRUE)        // Keep processing packets until there are no more
    {
      p = low_level_input (netif);                      // Get received packet and/or reset overflow


      if (!p) break;


      logPacket ("RX", p);


      if (netif->input (p, netif) != ERR_OK)            // Send packet to TCP/IP thread to process
      {
        LWIP_DEBUGF (NETIF_DEBUG, ("ethernetif_input: IP input error\n"));
        pbuf_free (p);
      }
    }
  }
}


/**
 * Allocates a pbuf and transfers the bytes of the incoming
 * packet from the Ethernet DMA buffer into the pbuf.  Also resets the DMA receive if
 * a receive overflow error occurred.
 *
 * @param netif the lwip network interface structure for this ethernetif
 * @return a pbuf filled with the received packet (including MAC header)
 *         NULL on memory error or no packet available
 */
static struct pbuf *
low_level_input (struct netif *netif)
{
  uint32_t length;
  uint8_t *buffer;
  struct pbuf *p = NULL, *q;
  int l = 0;


  // Descriptor is owned by DMA?  No data - return
  if (DMARxDescToGet->Status & ETH_DMARxDesc_OWN)
    return NULL;


  if (!(DMARxDescToGet->Status & ETH_DMARxDesc_ES)
      && (DMARxDescToGet->Status & ETH_DMARxDesc_LS)
      && (DMARxDescToGet->Status & ETH_DMARxDesc_FS))
  {
    // Frame Length of the received packet minus 4 bytes for the CRC */
    length = ((DMARxDescToGet->Status & ETH_DMARxDesc_FL) >> ETH_DMARxDesc_FrameLengthShift) - 4;
    buffer = (uint8_t *)(DMARxDescToGet->Buffer1Addr);    // Buffer address


#if ETH_PAD_SIZE
    length += ETH_PAD_SIZE;     // allow room for Ethernet padding
#endif


    /* We allocate a pbuf chain of pbufs from the pool. */
    p = pbuf_alloc (PBUF_RAW, length, PBUF_POOL);


    if (p != NULL)
    {
#if ETH_PAD_SIZE
      pbuf_header (p, -ETH_PAD_SIZE);   // drop the padding word
#endif


      // Iterate over the pbuf chain until the entire packet is copied into the pbuf
      for (q = p; q != NULL && l < length; q = q->next)
      { // Read enough bytes to fill this pbuf in the chain (q->len)
        memcpy ((uint8_t *)(q->payload), (uint8_t *)(&buffer[l]), q->len);
        l = l + q->len;
      }


#if ETH_PAD_SIZE
      pbuf_header (p, ETH_PAD_SIZE); /* reclaim the padding word */
#endif


      LINK_STATS_INC (link.recv);
    }
    else
    {
      LINK_STATS_INC (link.memerr);
      LINK_STATS_INC (link.drop);
    }
  }
  else
  {
    length = 0;
    LINK_STATS_INC (link.drop);
  }


  DMARxDescToGet->Status = ETH_DMARxDesc_OWN;   // Return ownership to Ethernet DMA


  if ((ETH->DMASR & ETH_DMASR_RBUS) != (uint32_t)RESET)  
  {
    ETH->DMASR = ETH_DMASR_RBUS;        // Clear RBUS ETHERNET DMA flag
    ETH->DMARPDR = 0;                   // Resume DMA reception
  }


  // Update the ETHERNET DMA global Rx descriptor with next Rx decriptor
  DMARxDescToGet = (ETH_DMADESCTypeDef *)(DMARxDescToGet->Buffer2NextDescAddr);


  return p;
}


/**
 * This function should do the actual transmission of the packet. The packet is
 * contained in the pbuf that is passed to the function. This pbuf
 * might be chained.
 *
 * @param netif the lwip network interface structure for this ethernetif
 * @param p the MAC packet to send (e.g. IP packet including MAC addresses and type)
 * @return ERR_OK if the packet could be sent
 *         an err_t value if the packet couldn't be sent
 *
 * @note Returning ERR_MEM here if a DMA queue of your MAC is full can lead to
 *       strange results. You might consider waiting for space in the DMA queue
 *       to become available since the stack doesn't retry to send a packet
 *       dropped because of memory failure (except for the TCP timers).
 */
static err_t
low_level_output (struct netif *netif, struct pbuf *p)
{
  uint8_t retrycount;
  struct pbuf *q;
  u32_t len = 0;
  uint8_t *buffer;


  xSemaphoreTake (ETH_TX_Mutex, portMAX_DELAY);         // Lock transmit subsystem


  for (retrycount = 0; retrycount < 20; retrycount++)
  {
    if (!(DMATxDescToSet->Status & ETH_DMATxDesc_OWN))  // Descriptor is available (not owned by DMA)?
      break;


    logMessage (LOG_LEVEL_DEBUG, "TXRTY");
    vTaskDelay (10 / portTICK_RATE_MS);
  }


  if (retrycount == 20)
  {
    xSemaphoreGive (ETH_TX_Mutex);                      // Unlock transmit subsystem
    return ERR_MEM;
  }


#if ETH_PAD_SIZE
  pbuf_header (p, -ETH_PAD_SIZE);                       // drop the padding word
#endif


  buffer = (uint8_t *)(DMATxDescToSet->Buffer1Addr);


  // Copy pbuf chain of data to DMA buffer
  for (q = p; q != NULL && len < p->tot_len; q = q->next)
  {
    memcpy ((uint8_t *)&buffer[len], (uint8_t *)q->payload, q->len);
    len += q->len;
  }


  DMATxDescToSet->ControlBufferSize = (len & ETH_DMATxDesc_TBS1);       // Set the Frame Length: bits[12:0]


  // Set to DMA owned and set the last segment and first segment bits (transmitted in one descriptor)
  DMATxDescToSet->Status |= ETH_DMATxDesc_LS | ETH_DMATxDesc_FS;
  DMATxDescToSet->Status |= ETH_DMATxDesc_OWN;


  /* When Tx Buffer unavailable flag is set: clear it and resume transmission */
  if (ETH->DMASR & ETH_DMASR_TBUS)
  {
    ETH->DMASR = ETH_DMASR_TBUS;        // Clear TBUS ETHERNET DMA flag
    ETH->DMATPDR = 0;                   // Resume DMA transmission
  }


  DMATxDescToSet = (ETH_DMADESCTypeDef *)(DMATxDescToSet->Buffer2NextDescAddr);         // Next descriptor


  xSemaphoreGive (ETH_TX_Mutex);        // Unlock transmit subsystem


  LINK_STATS_INC (link.xmit);


#if ETH_PAD_SIZE
  pbuf_header (p, ETH_PAD_SIZE);        // reclaim the padding word
#endif


  return ERR_OK;
}


/**
 * Configure the IO for Ethernet use.
 */
static void
ethGpioInit (void)
{
  GPIO_InitTypeDef gpioInit;


  /*
    ETHERNET pin configuration 
    AF Output Push Pull:
  - ETH_MII_MDIO / ETH_RMII_MDIO: PA2
  - ETH_MII_MDC / ETH_RMII_MDC: PC1
  - ETH_MII_TX_EN / ETH_RMII_TX_EN: PB11
  - ETH_MII_TXD0 / ETH_RMII_TXD0: PB12
  - ETH_MII_TXD1 / ETH_RMII_TXD1: PB13
  - MCO oscillator output: PA8


  Input:
  - ETH_MII_RX_CLK / ETH_RMII_REF_CLK: PA1
  - ETH_MII_RX_DV / ETH_RMII_CRS_DV: PA7
  - ETH_MII_RXD0 / ETH_RMII_RXD0: PC4
  - ETH_MII_RXD1 / ETH_RMII_RXD1: PC5
  */


  // Enable GPIOs clocks
  RCC_APB2PeriphClockCmd (RCC_APB2Periph_GPIOA | RCC_APB2Periph_GPIOB | RCC_APB2Periph_GPIOC
                          | RCC_APB2Periph_AFIO, ENABLE);


  /* Disable Ethernet I/O remapping */
  GPIO_PinRemapConfig (GPIO_Remap_ETH, DISABLE);


  /* Configure PA2 as alternate function push-pull */
  gpioInit.GPIO_Pin = GPIO_Pin_2;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  gpioInit.GPIO_Mode = GPIO_Mode_AF_PP;
  GPIO_Init(GPIOA, &gpioInit);


  /* Configure PC1 as alternate function push-pull */
  gpioInit.GPIO_Pin = GPIO_Pin_1;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  gpioInit.GPIO_Mode = GPIO_Mode_AF_PP;
  GPIO_Init(GPIOC, &gpioInit);


  /* Configure PB11, PB12 and PB13 as alternate function push-pull */
  gpioInit.GPIO_Pin =  GPIO_Pin_11 | GPIO_Pin_12 | GPIO_Pin_13;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  gpioInit.GPIO_Mode = GPIO_Mode_AF_PP;
  GPIO_Init(GPIOB, &gpioInit);


  /* Configure MCO (PA8) as alternate function push-pull */
  gpioInit.GPIO_Pin = GPIO_Pin_8;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  gpioInit.GPIO_Mode = GPIO_Mode_AF_PP;
  GPIO_Init(GPIOA, &gpioInit);


  /* Configure PA1 and PA7 as input */
  gpioInit.GPIO_Pin = GPIO_Pin_1 | GPIO_Pin_7 ;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  gpioInit.GPIO_Mode = GPIO_Mode_IN_FLOATING;
  GPIO_Init(GPIOA, &gpioInit);


  /* Configure PC4 and PC5 as input */
  gpioInit.GPIO_Pin = GPIO_Pin_4 | GPIO_Pin_5;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  gpioInit.GPIO_Mode = GPIO_Mode_IN_FLOATING;
  GPIO_Init(GPIOC, &gpioInit);
}


/**
 * Initialize all IO and peripherals required for Ethernet communication.
 *
 * @return pdSUCCESS if success, pdFAIL to signal some error.
 */
static portBASE_TYPE
ethMacInit (UINT8 *ucMACAddress)
{
  GPIO_InitTypeDef gpioInit;
  ETH_InitTypeDef xEthInit;
  NVIC_InitTypeDef xNVICInit;
  portBASE_TYPE xReturn;
  unsigned long ul;
  uint32_t start;


  // Section 29.4.4 of the STM32F107xx Reference Manual (CD00171190) says this should be done before enabling Ethernet clocks
  GPIO_ETH_MediaInterfaceConfig (GPIO_ETH_MediaInterface_RMII);          // RMII mode


  // Enable ETHERNET clocks
  RCC_AHBPeriphClockCmd (RCC_AHBPeriph_ETH_MAC | RCC_AHBPeriph_ETH_MAC_Tx |
                         RCC_AHBPeriph_ETH_MAC_Rx, ENABLE);


  ethGpioInit ();                       // Initialize Ethernet GPIO pins


  RCC_MCOConfig (RCC_MCO_XT1);          // Set MCO output to XT1 (raw 25MHz crystal oscillator input)


  // Configure PHY reset pin
  gpioInit.GPIO_Pin = GPIO_Pin_7;
  gpioInit.GPIO_Mode = GPIO_Mode_Out_PP;
  gpioInit.GPIO_Speed = GPIO_Speed_50MHz;
  GPIO_Init (GPIOE, &gpioInit);


  // Force a PHY reset (in case of system soft reset)
  GPIO_WriteBit (GPIOE, GPIO_Pin_7, Bit_RESET);


  // Delay to allow for proper reset
  vTaskDelay (ETHERNET_RESET_DELAY_MS / portTICK_PERIOD_MS);


  GPIO_WriteBit (GPIOE, GPIO_Pin_7, Bit_SET);   // Take PHY out of reset




  ETH_DeInit ();                        // Start with Ethernet in a known state
  ETH_SoftwareReset ();                 // Reset ethernet peripheral


  start = platformTimerStart ();


  // Wait for reset
  while (ETH_GetSoftwareResetStatus () == SET)
    if (platformTimerEllapsed (start) > ETH_RESET_TIMEOUT)
      return pdFAIL;


  /* Set the MAC address. */
  ETH_MACAddressConfig (ETH_MAC_Address0, ucMACAddress);


  /* Ethernet configuration structure */
  ETH_StructInit (&xEthInit);
  xEthInit.ETH_AutoNegotiation = ETH_AutoNegotiation_Enable;
  xEthInit.ETH_Watchdog = ETH_Watchdog_Disable;
  xEthInit.ETH_Jabber = ETH_Jabber_Disable;
  xEthInit.ETH_Mode = ETH_Mode_FullDuplex;
  xEthInit.ETH_ReceiveOwn = ETH_ReceiveOwn_Disable;
  xEthInit.ETH_RetryTransmission = ETH_RetryTransmission_Disable;
  xEthInit.ETH_ReceiveAll = ETH_ReceiveAll_Enable;
  xEthInit.ETH_PassControlFrames = ETH_PassControlFrames_ForwardPassedAddrFilter;
  xEthInit.ETH_TxDMABurstLength = ETH_TxDMABurstLength_32Beat;
  xEthInit.ETH_RxDMABurstLength = ETH_RxDMABurstLength_32Beat;
  xEthInit.ETH_DMAArbitration = ETH_DMAArbitration_RoundRobin_RxTx_1_1;
  xEthInit.ETH_DropTCPIPChecksumErrorFrame = ETH_DropTCPIPChecksumErrorFrame_Enable;
  xEthInit.ETH_ReceiveStoreForward = ETH_ReceiveStoreForward_Enable;
  xEthInit.ETH_TransmitStoreForward = ETH_TransmitStoreForward_Enable;


  // Initialize the Ethernet configuration
  xReturn = ETH_Init (&xEthInit, PHY_ADDRESS);


  // Was init successful?
  if (xReturn != pdFAIL)
  { // Enable Ethernet interrupts
    ETH_DMAITConfig (ETH_DMA_IT_R | ETH_DMA_IT_NIS, ENABLE);


    // Initialize Tx and Rx DMA chains
    ETH_DMATxDescChainInit (xTxDescriptors, (void *)txMACBuffers, NUM_TX_DESCRIPTORS);
    ETH_DMARxDescChainInit (xRxDescriptors, (void *)rxMACBuffers, NUM_RX_DESCRIPTORS);


    // Ensure received data generates an interrupt
    for (ul = 0; ul < NUM_RX_DESCRIPTORS; ul++)
      ETH_DMARxDescReceiveITConfig (&(xRxDescriptors[ul]), ENABLE);


    // TODO: Enable hardware checksums


    // Enable Ethernet interrupt with highest priority
    // NOTE: Priority is specified here as 0-15 (0 being highest priority)
    // NOTE: configMAX_SYSCALL_INTERRUPT_PRIORITY is specified as raw Cortex-M3 value (see FreeRTOSConfig.h)
    xNVICInit.NVIC_IRQChannel = ETH_IRQn;
    xNVICInit.NVIC_IRQChannelPreemptionPriority = configMAX_SYSCALL_INTERRUPT_PRIORITY >> 4;
    xNVICInit.NVIC_IRQChannelSubPriority = 0;
    xNVICInit.NVIC_IRQChannelCmd = ENABLE;
    NVIC_Init (&xNVICInit);


    ETH_Start ();               // Start ethernet hardware.
  }


  return xReturn;
}


/**
 * Ethernet IRQ handler.
 */
void
ETH_IRQHandler (void)
{
  long xHigherPriorityTaskWoken = pdFALSE;


  if (ETH->DMASR & ETH_DMASR_RS)        // Receive interrupt
    xSemaphoreGiveFromISR (ETH_RX_Sem, &xHigherPriorityTaskWoken);      // Signal receive semaphore


  ETH->DMASR = ETH_DMASR_RS;
  ETH->DMASR = ETH_DMASR_NIS;


  portEND_SWITCHING_ISR (xHigherPriorityTaskWoken);
}

Outcomes