diff --git a/src/Makefile.am b/src/Makefile.am index 6acbd398ef..93ab1c6e6e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -13,6 +13,7 @@ source-pcap.c source-pcap.h \ source-pcap-file.c source-pcap-file.h \ source-pfring.c source-pfring.h \ source-ipfw.c source-ipfw.h \ +source-erf-file.c source-erf-file.h \ decode.c decode.h \ decode-ethernet.c decode-ethernet.h \ decode-vlan.c decode-vlan.h \ diff --git a/src/runmodes.c b/src/runmodes.c index f05eff3c0a..274ec10480 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -2732,3 +2732,119 @@ int RunModeIdsPfringAuto(DetectEngineCtx *de_ctx, char *iface) { return 0; } +int RunModeErfFileAuto(DetectEngineCtx *de_ctx, char *file) +{ + SCEnter(); + char tname[12]; + uint16_t cpu = 0; + + /* Available cpus */ + uint16_t ncpus = UtilCpuGetNumProcessorsOnline(); + + SCLogDebug("file %s", file); + TimeModeSetOffline(); + + /* create the threads */ + ThreadVars *tv_receiveerf = TmThreadCreatePacketHandler("ReceiveErfFile", + "packetpool","packetpool","pickup-queue","simple","1slot"); + if (tv_receiveerf == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(EXIT_FAILURE); + } + TmModule *tm_module = TmModuleGetByName("ReceiveErfFile"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed for ReceiveErfFile\n"); + exit(EXIT_FAILURE); + } + Tm1SlotSetFunc(tv_receiveerf, tm_module, file); + + TmThreadSetCPUAffinity(tv_receiveerf, 0); + if (ncpus > 1) + TmThreadSetThreadPriority(tv_receiveerf, PRIO_MEDIUM); + + if (TmThreadSpawn(tv_receiveerf) != TM_ECODE_OK) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + ThreadVars *tv_decode1 = TmThreadCreatePacketHandler("Decode & Stream", + "pickup-queue","simple","stream-queue1","simple","varslot"); + if (tv_decode1 == NULL) { + printf("ERROR: TmThreadsCreate failed for Decode1\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("DecodeErfFile"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeErfFile failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv_decode1,tm_module,NULL); + + tm_module = TmModuleGetByName("StreamTcp"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv_decode1,tm_module,NULL); + + TmThreadSetCPUAffinity(tv_decode1, 0); + if (ncpus > 1) + TmThreadSetThreadPriority(tv_decode1, PRIO_MEDIUM); + + if (TmThreadSpawn(tv_decode1) != TM_ECODE_OK) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + for (cpu = 0; cpu < ncpus; cpu++) { + snprintf(tname, sizeof(tname),"Detect%"PRIu16, cpu+1); + if (tname == NULL) + break; + + char *thread_name = SCStrdup(tname); + SCLogDebug("Assigning %s affinity to cpu %u", thread_name, cpu); + + ThreadVars *tv_detect_ncpu = TmThreadCreatePacketHandler(thread_name,"stream-queue1","simple","alert-queue1","simple","1slot"); + if (tv_detect_ncpu == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName Detect failed\n"); + exit(EXIT_FAILURE); + } + Tm1SlotSetFunc(tv_detect_ncpu,tm_module,(void *)de_ctx); + + TmThreadSetCPUAffinity(tv_detect_ncpu, (int)cpu); + /* If we have more than one core/cpu, the first Detect thread + * (at cpu 0) will have less priority (higher 'nice' value) + * In this case we will set the thread priority to +10 (default is 0) + */ + if (cpu == 0 && ncpus > 1) { + TmThreadSetThreadPriority(tv_detect_ncpu, PRIO_LOW); + } else if (ncpus > 1) { + TmThreadSetThreadPriority(tv_detect_ncpu, PRIO_MEDIUM); + } + + if (TmThreadSpawn(tv_detect_ncpu) != TM_ECODE_OK) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + } + + ThreadVars *tv_outputs = TmThreadCreatePacketHandler("Outputs", + "alert-queue1", "simple", "packetpool", "packetpool", "varslot"); + SetupOutputs(tv_outputs); + + TmThreadSetCPUAffinity(tv_outputs, 0); + if (ncpus > 1) + TmThreadSetThreadPriority(tv_outputs, PRIO_MEDIUM); + + if (TmThreadSpawn(tv_outputs) != TM_ECODE_OK) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/src/runmodes.h b/src/runmodes.h index 57f5345b28..8ec7a673c6 100644 --- a/src/runmodes.h +++ b/src/runmodes.h @@ -46,6 +46,8 @@ int RunModeIdsPfringAuto(DetectEngineCtx *, char *); int RunModeIpsIPFW(DetectEngineCtx *); int RunModeIpsIPFWAuto(DetectEngineCtx *); +int RunModeErfFileAuto(DetectEngineCtx *, char *); + void RunModeShutDown(void); #endif /* __RUNMODES_H__ */ diff --git a/src/source-erf-file.c b/src/source-erf-file.c new file mode 100644 index 0000000000..e48d22960f --- /dev/null +++ b/src/source-erf-file.c @@ -0,0 +1,255 @@ +/* Copyright (C) 2010 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Endace Technology Limited. + * + * Support for reading ERF files. + * + * Only ethernet supported at this time. + */ + +#include "suricata-common.h" +#include "suricata.h" +#include "tm-modules.h" + +#define DAG_TYPE_ETH 2 + +typedef struct DagFlags_ { + uint8_t iface:2; + uint8_t vlen:1; + uint8_t trunc:1; + uint8_t rxerror:1; + uint8_t dserror:1; + uint8_t reserved:1; + uint8_t direction:1; +} DagFlags; + +typedef struct DagRecord_ { + uint64_t ts; + uint8_t type; + DagFlags flags; + uint16_t rlen; + uint16_t lctr; + uint16_t wlen; + uint16_t pad; +} __attribute__((packed)) DagRecord; + +typedef struct ErfFileThreadVars_ { + FILE *erf; + ThreadVars *tv; + + uint32_t pkts; + uint64_t bytes; +} ErfFileThreadVars; + +TmEcode ReceiveErfFile(ThreadVars *, Packet *, void *, PacketQueue *); +TmEcode ReceiveErfFileThreadInit(ThreadVars *, void *, void **); +void ReceiveErfFileThreadExitStats(ThreadVars *, void *); +TmEcode ReceiveErfFileThreadDeinit(ThreadVars *, void *); + +TmEcode DecodeErfFileThreadInit(ThreadVars *, void *, void **); +TmEcode DecodeErfFile(ThreadVars *, Packet *, void *, PacketQueue *); + +/** + * \brief Register the ERF file receiver (reader) module. + */ +void +TmModuleReceiveErfFileRegister(void) +{ + tmm_modules[TMM_RECEIVEERFFILE].name = "ReceiveErfFile"; + tmm_modules[TMM_RECEIVEERFFILE].ThreadInit = ReceiveErfFileThreadInit; + tmm_modules[TMM_RECEIVEERFFILE].Func = ReceiveErfFile; + tmm_modules[TMM_RECEIVEERFFILE].ThreadExitPrintStats = + ReceiveErfFileThreadExitStats; + tmm_modules[TMM_RECEIVEERFFILE].ThreadDeinit = NULL; + tmm_modules[TMM_RECEIVEERFFILE].RegisterTests = NULL; + tmm_modules[TMM_RECEIVEERFFILE].cap_flags = 0; +} + +/** + * \brief Register the ERF file decoder module. + */ +void +TmModuleDecodeErfFileRegister(void) +{ + tmm_modules[TMM_DECODEERFFILE].name = "DecodeErfFile"; + tmm_modules[TMM_DECODEERFFILE].ThreadInit = DecodeErfFileThreadInit; + tmm_modules[TMM_DECODEERFFILE].Func = DecodeErfFile; + tmm_modules[TMM_DECODEERFFILE].ThreadExitPrintStats = NULL; + tmm_modules[TMM_DECODEERFFILE].ThreadDeinit = NULL; + tmm_modules[TMM_DECODEERFFILE].RegisterTests = NULL; + tmm_modules[TMM_DECODEERFFILE].cap_flags = 0; +} + +/** + * \brief Thread entry function for ERF reading. + * + * Reads a new ERF record from the file and sets up the Packet for + * decoding. + */ +TmEcode +ReceiveErfFile(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) +{ + SCEnter(); + + ErfFileThreadVars *etv = (ErfFileThreadVars *)data; + DagRecord dr; + + int r = fread(&dr, sizeof(DagRecord), 1, etv->erf); + if (r < 1) { + SCLogInfo("End of ERF file reached or an error occurred."); + EngineStop(); + SCReturnInt(TM_ECODE_FAILED); + } + int rlen = ntohs(dr.rlen); + int wlen = ntohs(dr.wlen); + r = fread(p->pkt, rlen - sizeof(DagRecord), 1, etv->erf); + if (r < 1) { + SCLogInfo("End of ERF file reached or an error occurred."); + EngineStop(); + SCReturnInt(TM_ECODE_FAILED); + } + + /* Only support ethernet at this time. */ + if (dr.type != DAG_TYPE_ETH) { + SCLogError(SC_ERR_UNIMPLEMENTED, + "DAG record type %d not implemented.", dr.type); + SCReturnInt(TM_ECODE_FAILED); + } + + p->pktlen = wlen - 4; /* Trim the FCS... */ + p->datalink = LINKTYPE_ETHERNET; + + /* Convert ERF time to timeval - from libpcap. */ + uint64_t ts = dr.ts; + p->ts.tv_sec = ts >> 32; + ts = (ts & 0xffffffffULL) * 1000000; + ts += 0x80000000; /* rounding */ + p->ts.tv_usec = ts >> 32; + if (p->ts.tv_usec >= 1000000) { + p->ts.tv_usec -= 1000000; + p->ts.tv_sec++; + } + + etv->pkts++; + etv->bytes += wlen; + + SCReturnInt(TM_ECODE_OK); +} + +/** + * \brief Initialize the ERF receiver thread. + */ +TmEcode +ReceiveErfFileThreadInit(ThreadVars *tv, void *initdata, void **data) +{ + SCEnter(); + + if (initdata == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Error: No filename provided."); + SCReturnInt(TM_ECODE_FAILED); + } + + FILE *erf = fopen((const char *)initdata, "r"); + if (erf == NULL) { + SCLogError(SC_ERR_FOPEN, "Failed to open %s: %s", (char *)initdata, + strerror(errno)); + exit(EXIT_FAILURE); + } + + ErfFileThreadVars *etv = SCMalloc(sizeof(ErfFileThreadVars)); + if (etv == NULL) { + SCLogError(SC_ERR_MEM_ALLOC, + "Failed to allocate memory for ERF file thread vars."); + fclose(erf); + SCReturnInt(TM_ECODE_FAILED); + } + memset(etv, 0, sizeof(*etv)); + etv->erf = erf; + etv->tv = tv; + *data = (void *)etv; + + SCLogInfo("Processing ERF file %s", (char *)initdata); + + SCReturnInt(TM_ECODE_OK); +} + +/** + * \brief Initialize the ERF decoder thread. + */ +TmEcode +DecodeErfFileThreadInit(ThreadVars *tv, void *initdata, void **data) +{ + SCEnter(); + DecodeThreadVars *dtv = NULL; + + if ((dtv = SCMalloc(sizeof(DecodeThreadVars))) == NULL) + SCReturnInt(TM_ECODE_FAILED); + memset(dtv, 0, sizeof(DecodeThreadVars)); + + DecodeRegisterPerfCounters(dtv, tv); + + *data = (void *)dtv; + + SCReturnInt(TM_ECODE_OK); +} + +/** + * \brief Decode the ERF file. + * + * This function ups the decoder counters and then passes the packet + * off to the ethernet decoder. + */ +TmEcode +DecodeErfFile(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) +{ + SCEnter(); + DecodeThreadVars *dtv = (DecodeThreadVars *)data; + + /* Update counters. */ + SCPerfCounterIncr(dtv->counter_pkts, tv->sc_perf_pca); + SCPerfCounterIncr(dtv->counter_pkts_per_sec, tv->sc_perf_pca); + + SCPerfCounterAddUI64(dtv->counter_bytes, tv->sc_perf_pca, p->pktlen); + SCPerfCounterAddDouble(dtv->counter_bytes_per_sec, tv->sc_perf_pca, p->pktlen); + SCPerfCounterAddDouble(dtv->counter_mbit_per_sec, tv->sc_perf_pca, + (p->pktlen * 8)/1000000.0 ); + + SCPerfCounterAddUI64(dtv->counter_avg_pkt_size, tv->sc_perf_pca, p->pktlen); + SCPerfCounterSetUI64(dtv->counter_max_pkt_size, tv->sc_perf_pca, p->pktlen); + + DecodeEthernet(tv, dtv, p, p->pkt, p->pktlen, pq); + + SCReturnInt(TM_ECODE_OK); +} + +/** + * \brief Print some stats to the log at program exit. + * + * \param tv Pointer to ThreadVars. + * \param data Pointer to data, ErfFileThreadVars. + */ +void +ReceiveErfFileThreadExitStats(ThreadVars *tv, void *data) +{ + ErfFileThreadVars *etv = (ErfFileThreadVars *)data; + + SCLogInfo("Packets: %"PRIu32"; Bytes: %"PRIu64, etv->pkts, etv->bytes); +} diff --git a/src/source-erf-file.h b/src/source-erf-file.h new file mode 100644 index 0000000000..fc56f74385 --- /dev/null +++ b/src/source-erf-file.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2010 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Endace Technology Limited + */ + +#ifndef __SOURCE_ERF_H__ +#define __SOURCE_ERF_H__ + +void TmModuleReceiveErfFileRegister(void); +void TmModuleDecodeErfFileRegister(void); + +#endif /* __SOURCE_ERF_H__ */ diff --git a/src/suricata.c b/src/suricata.c index 3b3ee3be33..f55ffce4d7 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -83,6 +83,8 @@ #include "source-pfring.h" +#include "source-erf-file.h" + #include "respond-reject.h" #include "flow.h" @@ -301,6 +303,7 @@ void usage(const char *progname) printf("\t--user : run suricata as this user after init\n"); printf("\t--group : run suricata as this group after init\n"); #endif /* HAVE_LIBCAP_NG */ + printf("\t--erf-in : process an ERF file\n"); printf("\n"); printf("\nTo run the engine with default configuration on " "interface eth0 with signature file \"signatures.rules\", run the " @@ -330,6 +333,7 @@ int main(int argc, char **argv) uint8_t do_setgid = FALSE; uint32_t userid = 0; uint32_t groupid = 0; + char *erf_file = NULL; char *log_dir; struct stat buf; @@ -364,6 +368,7 @@ int main(int argc, char **argv) {"fatal-unittests", 0, 0, 0}, {"user", required_argument, 0, 0}, {"group", required_argument, 0, 0}, + {"erf-in", required_argument, 0, 0}, {NULL, 0, NULL, 0} }; @@ -458,6 +463,10 @@ int main(int argc, char **argv) do_setgid = TRUE; #endif /* HAVE_LIBCAP_NG */ } + else if (strcmp((long_opts[option_index]).name, "erf-in") == 0) { + run_mode = MODE_ERF_FILE; + erf_file = optarg; + } break; case 'c': conf_filename = optarg; @@ -687,6 +696,8 @@ int main(int argc, char **argv) #ifdef __SC_CUDA_SUPPORT__ TmModuleCudaMpmB2gRegister(); #endif + TmModuleReceiveErfFileRegister(); + TmModuleDecodeErfFileRegister(); TmModuleDebugList(); /** \todo we need an api for these */ @@ -921,6 +932,9 @@ int main(int argc, char **argv) //RunModeIpsIPFW(de_ctx); RunModeIpsIPFWAuto(de_ctx); } + else if (run_mode == MODE_ERF_FILE) { + RunModeErfFileAuto(de_ctx, erf_file); + } else { SCLogError(SC_ERR_UNKNOWN_RUN_MODE, "Unknown runtime mode. Aborting"); exit(EXIT_FAILURE); diff --git a/src/suricata.h b/src/suricata.h index bbbf43c6af..9f019ac327 100644 --- a/src/suricata.h +++ b/src/suricata.h @@ -53,7 +53,8 @@ enum { MODE_PFRING, MODE_NFQ, MODE_IPFW, - MODE_UNITTEST + MODE_UNITTEST, + MODE_ERF_FILE, }; /* preallocated packet structures here diff --git a/src/tm-modules.h b/src/tm-modules.h index cbe920f3c0..2cacbb7f4b 100644 --- a/src/tm-modules.h +++ b/src/tm-modules.h @@ -79,6 +79,8 @@ enum { #ifdef __SC_CUDA_SUPPORT__ TMM_CUDA_MPM_B2G, #endif + TMM_RECEIVEERFFILE, + TMM_DECODEERFFILE, TMM_SIZE, };