You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
suricata/src/source-pcap-file-helper.h

125 lines
3.3 KiB
C

pcap/file: fix race during pcap processing start A race condition during the start of pcap file processing could cause missed alerts and logged events. This race happens between the packet threads and the flow manager. It was observed on slower hardware, but in theory could happen on any machine. It required the 'autofp' runmode. In commit 6f560144c1b9 ("time: improve offline time handling") the logic was added to make the flow manager use a minimum of all the packet threads perception of time. The race condition was that the flow manager may become active _before_ all of the packet threads have started processing packets and thus setting their timestamp. The threads that had not yet initialized their timestamp would not be considered when calculating the minimum. As a result of this, older packets timestamps would not yet be registered. This would give the Flow Manager a timestamp too far in the future. While the FM was running, the packet processing would start and a flow would be created. This flow would then immediately be considered 'timed out' by the FM, due to the timestamp too far in the future. In the observed case, the thread processing packet 1 from the pcap had not yet started processing while other threads had already started. The FM was also already active. Due to the timestamps in the pcap this meant that the time the FM used was about 500 seconds in the future compared to packet 1. This patch fixes the issue by initializing all of the threads timestamps with the timestamp value of the first packet. This way the minimum will always consider this timestamp.
5 years ago
/* Copyright (C) 2007-2020 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Danny Browning <danny.browning@protectwise.com>
*/
#include "suricata-common.h"
#include "tm-threads.h"
#ifndef SURICATA_SOURCE_PCAP_FILE_HELPER_H
#define SURICATA_SOURCE_PCAP_FILE_HELPER_H
typedef struct PcapFileGlobalVars_ {
uint64_t cnt; /** packet counter */
ChecksumValidationMode conf_checksum_mode;
ChecksumValidationMode checksum_mode;
SC_ATOMIC_DECLARE(unsigned int, invalid_checksums);
uint32_t read_buffer_size;
} PcapFileGlobalVars;
/**
* Data that is shared amongst File, Directory, and Thread level vars
*/
typedef struct PcapFileSharedVars_
{
char *bpf_string;
uint32_t tenant_id;
struct timespec last_processed;
bool should_delete;
ThreadVars *tv;
TmSlot *slot;
/* counters */
uint64_t pkts;
uint64_t bytes;
uint64_t files;
uint8_t done;
uint32_t errs;
/** callback result -- set if one of the thread module failed. */
int cb_result;
} PcapFileSharedVars;
/**
* Data specific to a single pcap file
*/
typedef struct PcapFileFileVars_
{
char *filename;
pcap_t *pcap_handle;
int datalink;
struct bpf_program filter;
PcapFileSharedVars *shared;
pcap/file: fix race during pcap processing start A race condition during the start of pcap file processing could cause missed alerts and logged events. This race happens between the packet threads and the flow manager. It was observed on slower hardware, but in theory could happen on any machine. It required the 'autofp' runmode. In commit 6f560144c1b9 ("time: improve offline time handling") the logic was added to make the flow manager use a minimum of all the packet threads perception of time. The race condition was that the flow manager may become active _before_ all of the packet threads have started processing packets and thus setting their timestamp. The threads that had not yet initialized their timestamp would not be considered when calculating the minimum. As a result of this, older packets timestamps would not yet be registered. This would give the Flow Manager a timestamp too far in the future. While the FM was running, the packet processing would start and a flow would be created. This flow would then immediately be considered 'timed out' by the FM, due to the timestamp too far in the future. In the observed case, the thread processing packet 1 from the pcap had not yet started processing while other threads had already started. The FM was also already active. Due to the timestamps in the pcap this meant that the time the FM used was about 500 seconds in the future compared to packet 1. This patch fixes the issue by initializing all of the threads timestamps with the timestamp value of the first packet. This way the minimum will always consider this timestamp.
5 years ago
/* fields used to get the first packet's timestamp early,
pcap/file: fix race during pcap processing start A race condition during the start of pcap file processing could cause missed alerts and logged events. This race happens between the packet threads and the flow manager. It was observed on slower hardware, but in theory could happen on any machine. It required the 'autofp' runmode. In commit 6f560144c1b9 ("time: improve offline time handling") the logic was added to make the flow manager use a minimum of all the packet threads perception of time. The race condition was that the flow manager may become active _before_ all of the packet threads have started processing packets and thus setting their timestamp. The threads that had not yet initialized their timestamp would not be considered when calculating the minimum. As a result of this, older packets timestamps would not yet be registered. This would give the Flow Manager a timestamp too far in the future. While the FM was running, the packet processing would start and a flow would be created. This flow would then immediately be considered 'timed out' by the FM, due to the timestamp too far in the future. In the observed case, the thread processing packet 1 from the pcap had not yet started processing while other threads had already started. The FM was also already active. Due to the timestamps in the pcap this meant that the time the FM used was about 500 seconds in the future compared to packet 1. This patch fixes the issue by initializing all of the threads timestamps with the timestamp value of the first packet. This way the minimum will always consider this timestamp.
5 years ago
* so it can be used to setup the time subsys. */
const u_char *first_pkt_data;
struct pcap_pkthdr *first_pkt_hdr;
struct timeval first_pkt_ts;
/** flex array member for the libc io read buffer. Size controlled by
* PcapFileGlobalVars::read_buffer_size. */
#if defined(HAVE_SETVBUF) && defined(OS_LINUX)
char buffer[];
#endif
} PcapFileFileVars;
/**
* Dispatch a file for processing, where the information necessary to process that
* file is as PcapFileFileVars object.
* @param ptv PcapFileFileVars object to be processed
* @return
*/
TmEcode PcapFileDispatch(PcapFileFileVars *ptv);
/**
* From a PcapFileFileVars, prepare the filename for processing by setting
* pcap_handle, datalink, and filter
* @param pfv PcapFileFileVars object to populate
* @return
*/
TmEcode InitPcapFile(PcapFileFileVars *pfv);
/**
* Cleanup resources associated with a PcapFileFileVars object.
* @param pfv Object to be cleaned up
*/
void CleanupPcapFileFileVars(PcapFileFileVars *pfv);
/**
* Determine if a datalink type is valid, setting a decoder function if valid.
* @param datalink Datalink type to validate
* @param decoder Pointer to decoder to set if valid
* @return TM_ECODE_OK if valid datalink type and decoder has been set.
*/
TmEcode ValidateLinkType(int datalink, DecoderFunc *decoder);
const char *PcapFileGetFilename(void);
#endif /* SURICATA_SOURCE_PCAP_FILE_HELPER_H */