source/erf: Handle ERF META and PAD record types and extension headers

Add support for ERF record types META and PAD, plus additional ETH types.

Add support for ERF extension headers.

These types and extension headers were already supported by erf-source-dag.

Made source-erf-file and source-erf-dag more consistent.

Ticket: 8962

src/source-erf-dag.c

@@ -89,6 +89,9 @@ NoErfDagSupportExit(ThreadVars *tv, const void *initdata, void **data)
 /* Number of bytes per loop to process before fetching more data. */
 #define BYTES_PER_LOOP (4 * 1024 * 1024) /* 4 MB */
 
+#define ERF_EXT_LEN     8
+#define ERF_ETH_PAD_LEN 2
+
 extern uint32_t max_pending_packets;
 
 typedef struct ErfDagThreadVars_ {
@@ -394,6 +397,10 @@ ProcessErfDagRecords(ErfDagThreadVars *ewtn, uint8_t *top, uint32_t *pkts_read)
     while (((top - ewtn->btm) >= dag_record_size) &&
         ((processed + dag_record_size) < BYTES_PER_LOOP)) {
 
+        if (suricata_ctl_flags & SURICATA_STOP) {
+            SCReturnInt(TM_ECODE_OK);
+        }
+
         /* Make sure we have at least one packet in the packet pool,
          * to prevent us from alloc'ing packets at line rate. */
         PacketPoolWait();
@@ -419,7 +426,7 @@ ProcessErfDagRecords(ErfDagThreadVars *ewtn, uint8_t *top, uint32_t *pkts_read)
         processed += rlen;
 
         /* Only support ethernet at this time. */
-        switch (hdr_type & 0x7f) {
+        switch (hdr_type & ERF_TYPE_MASK) {
             case ERF_TYPE_PAD:
             case ERF_TYPE_META:
                 /* Skip. */
@@ -461,10 +468,10 @@ ProcessErfDagRecord(ErfDagThreadVars *ewtn, char *prec)
 {
     SCEnter();
 
-    int wlen = 0;
+    uint16_t wlen = 0;
-    int rlen = 0;
+    uint16_t rlen = 0;
     int hdr_num = 0;
-    char hdr_type = 0;
+    uint8_t hdr_type = 0;
     dag_record_t  *dr = (dag_record_t*)prec;
     erf_payload_t *pload;
     Packet *p;
@@ -474,23 +481,23 @@ ProcessErfDagRecord(ErfDagThreadVars *ewtn, char *prec)
     rlen = SCNtohs(dr->rlen);
 
     /* count extension headers */
-    while (hdr_type & 0x80) {
+    while (hdr_type & ERF_TYPE_MORE_EXT) {
-        if (rlen < (dag_record_size + (hdr_num * 8))) {
+        if (rlen < (dag_record_size + (hdr_num * ERF_EXT_LEN))) {
             SCLogError("Insufficient captured packet length.");
             SCReturnInt(TM_ECODE_FAILED);
         }
-        hdr_type = prec[(dag_record_size + (hdr_num * 8))];
+        hdr_type = prec[(dag_record_size + (hdr_num * ERF_EXT_LEN))];
         hdr_num++;
     }
 
     /* Check that the whole frame was captured */
-    if (rlen < (dag_record_size + (8 * hdr_num) + 2 + wlen)) {
+    if (rlen < (dag_record_size + (hdr_num * ERF_EXT_LEN) + ERF_ETH_PAD_LEN + wlen)) {
         SCLogInfo("Incomplete frame captured.");
         SCReturnInt(TM_ECODE_OK);
     }
 
     /* skip over extension headers */
-    pload = (erf_payload_t *)(prec + dag_record_size + (8 * hdr_num));
+    pload = (erf_payload_t *)(prec + dag_record_size + (hdr_num * ERF_EXT_LEN));
 
     p = PacketGetFromQueueOrAlloc();
     if (p == NULL) {

src/source-erf-file.c

@@ -31,7 +31,21 @@
 #include "source-erf-file.h"
 #include "util-datalink.h"
 
-#define DAG_TYPE_ETH 2
+#define ERF_HEADER_LEN  16
+#define ERF_EXT_LEN     8
+#define ERF_ETH_PAD_LEN 2
+
+#ifndef HAVE_DAG
+#define ERF_TYPE_MASK           0x7f
+#define ERF_TYPE_MORE_EXT       0x80
+#define ERF_TYPE_ETH            2
+#define ERF_TYPE_COLOR_ETH      11
+#define ERF_TYPE_DSM_COLOR_ETH  16
+#define ERF_TYPE_COLOR_HASH_ETH 20
+
+#else /* Implied we do have DAG support */
+#include <dagapi.h>
+#endif
 
 typedef struct DagFlags_ {
     uint8_t iface:2;
@@ -50,7 +64,6 @@ typedef struct DagRecord_ {
     uint16_t rlen;
     uint16_t lctr;
     uint16_t wlen;
-    uint16_t pad;
 } __attribute__((packed)) DagRecord;
 
 typedef struct ErfFileThreadVars_ {
@@ -131,7 +144,9 @@ TmEcode ReceiveErfFileLoop(ThreadVars *tv, void *data, void *slot)
          * to prevent us from alloc'ing packets at line rate. */
         PacketPoolWait();
 
+        if (p == NULL) {
             p = PacketGetFromQueueOrAlloc();
+        }
         if (unlikely(p == NULL)) {
             SCLogError("Failed to allocate a packet.");
             EngineStop();
@@ -144,11 +159,15 @@ TmEcode ReceiveErfFileLoop(ThreadVars *tv, void *data, void *slot)
             EngineStop();
             SCReturnInt(TM_ECODE_FAILED);
         }
+        if (GET_PKT_LEN(p) == 0) {
+            continue;
+        }
 
         if (TmThreadsSlotProcessPkt(etv->tv, etv->slot, p) != TM_ECODE_OK) {
             EngineStop();
             SCReturnInt(TM_ECODE_FAILED);
         }
+        p = NULL;
     }
     SCReturnInt(TM_ECODE_FAILED);
 }
@@ -159,6 +178,8 @@ static inline TmEcode ReadErfRecord(ThreadVars *tv, Packet *p, void *data)
 
     ErfFileThreadVars *etv = (ErfFileThreadVars *)data;
     DagRecord dr;
+    unsigned int hdr_num = 0;
+    char ext_hdr[ERF_EXT_LEN];
 
     size_t r = fread(&dr, sizeof(DagRecord), 1, etv->erf);
     if (r < 1) {
@@ -170,19 +191,21 @@ static inline TmEcode ReadErfRecord(ThreadVars *tv, Packet *p, void *data)
         }
         SCReturnInt(TM_ECODE_FAILED);
     }
+    uint8_t hdr_type = dr.type;
     uint16_t rlen = SCNtohs(dr.rlen);
-    if (rlen < sizeof(DagRecord)) {
+    if (rlen < ERF_HEADER_LEN) {
         SCLogError("Bad ERF record, "
                    "record length less than size of header");
         SCReturnInt(TM_ECODE_FAILED);
     }
 
-    uint32_t caplen = rlen - sizeof(DagRecord);
+    /* count extension headers */
-    if (caplen > MAX_PACKET_SIZE) {
+    while (hdr_type & ERF_TYPE_MORE_EXT) {
-        SCLogError("Bad ERF record, capture length %u exceeds max %d", caplen, MAX_PACKET_SIZE);
+        if (rlen < (ERF_HEADER_LEN + ((hdr_num + 1) * 8))) {
+            SCLogError("Insufficient captured packet length.");
             SCReturnInt(TM_ECODE_FAILED);
         }
-    r = fread(etv->buffer, caplen, 1, etv->erf);
+        r = fread(ext_hdr, ERF_EXT_LEN, 1, etv->erf);
         if (r < 1) {
             if (feof(etv->erf)) {
                 SCLogInfo("End of ERF file reached");
@@ -191,6 +214,31 @@ static inline TmEcode ReadErfRecord(ThreadVars *tv, Packet *p, void *data)
             }
             SCReturnInt(TM_ECODE_FAILED);
         }
+        hdr_type = ext_hdr[0];
+        hdr_num++;
+    }
+
+    /* read and discard ERF Ethernet pad */
+    if (rlen < (ERF_HEADER_LEN + (hdr_num * ERF_EXT_LEN) + ERF_ETH_PAD_LEN)) {
+        SCLogError("Insufficient captured packet length.");
+        SCReturnInt(TM_ECODE_FAILED);
+    }
+    r = fread(ext_hdr, ERF_ETH_PAD_LEN, 1, etv->erf);
+    if (r < 1) {
+        if (feof(etv->erf)) {
+            SCLogInfo("End of ERF file reached");
+        } else {
+            SCLogInfo("Error reading ERF record");
+        }
+        SCReturnInt(TM_ECODE_FAILED);
+    }
+
+    uint32_t caplen = rlen - (hdr_num * ERF_EXT_LEN) - ERF_HEADER_LEN - ERF_ETH_PAD_LEN;
+    if (caplen > MAX_PACKET_SIZE) {
+        SCLogError("Bad ERF record, capture length %u exceeds max %d", caplen, MAX_PACKET_SIZE);
+        SCReturnInt(TM_ECODE_FAILED);
+    }
+
     r = fread(etv->buffer, caplen, 1, etv->erf);
     if (r < 1) {
         if (feof(etv->erf)) {
@@ -202,10 +250,16 @@ static inline TmEcode ReadErfRecord(ThreadVars *tv, Packet *p, void *data)
         SCReturnInt(TM_ECODE_FAILED);
     }
 
-    /* Only support ethernet at this time. */
+    /* Only support ethernet at this time. Return TM_ECODE_OK with pkt len = 0 to indicate skipped
-    if (dr.type != DAG_TYPE_ETH) {
+     * record */
-        SCLogError("DAG record type %d not implemented.", dr.type);
+    switch (dr.type & ERF_TYPE_MASK) {
-        SCReturnInt(TM_ECODE_FAILED);
+        case ERF_TYPE_DSM_COLOR_ETH:
+        case ERF_TYPE_COLOR_ETH:
+        case ERF_TYPE_COLOR_HASH_ETH:
+        case ERF_TYPE_ETH:
+            break;
+        default:
+            SCReturnInt(TM_ECODE_OK);
     }
 
     if (PacketCopyData(p, etv->buffer, caplen) != 0) {