source-pcap-file: delete when done (2417)

https://redmine.openinfosecfoundation.org/issues/2417

Add option to have pcap files deleted after they have been processed.
This option combines well with pcap file continuous and streaming
files to a directory being processed.

doc/userguide/partials/options.rst

@@ -25,9 +25,9 @@
 
 .. option:: -r <path>
 
-   Run in pcap offline mode reading files from pcap file. If <path> specifies
-   a directory, all files in that directory will be processed in order of
-   modified time maintaining flow state between files.
+   Run in pcap offline mode (replay mode) reading files from pcap file. If
+   <path> specifies a directory, all files in that directory will be processed
+   in order of modified time maintaining flow state between files.
 
 .. option:: --pcap-file-continuous
 
@@ -35,6 +35,13 @@
    interrupted. This is useful with directories to add new files and not reset
    flow state between files.
 
+.. option:: --pcap-file-delete
+
+   Used with the -r option to indicate that the mode should delete pcap files
+   after they have been processed. This is useful with pcap-file-continuous to
+   continuously feed files to a directory and have them cleaned up when done. If
+   this option is not set, pcap files will not be deleted after processing.
+
 .. option::  -i <interface>
 
    After the -i option you can enter the interface card you would like

doc/userguide/unix-socket.rst

@@ -256,9 +256,37 @@ In pcap-file mode, this gives:
   SND: {"command": "pcap-file-list"}
   RCV: {"message": {"count": 1, "files": ["/home/eric/git/oisf/benches/sandnet.pcap"]}, "return": "OK"}
   Success: {'count': 1, 'files': ['/home/eric/git/oisf/benches/sandnet.pcap']}
+  >>> pcap-file-continuous /home/eric/git/oisf/benches /tmp/bench 0 true
+  SND: {"command": "pcap-file", "arguments": {"output-dir": "/tmp/bench", "filename": "/home/eric/git/oisf/benches/sandnet.pcap", "tenant": 0, "delete-when-done": true}}
+  RCV: {"message": "Successfully added file to list", "return": "OK"}
+  Success: Successfully added file to list
 
 There is one thing to be careful about: a Suricata message is sent in
 multiple send operations. This result in possible incomplete read on
 client side. The worse workaround is to sleep a bit before trying a
 recv call. An other solution is to use non blocking socket and retry a
 recv if the previous one has failed.
+
+Pcap-file json format is:
+
+::
+
+  {
+    "command": "pcap-file",
+    "arguments": {
+      "output-dir": "path to output dir",
+      "filename": "path to file or directory to run",
+      "tenant": 0,
+      "continuous": false,
+      "delete-when-done": false
+    }
+  }
+
+`output-dir` and `filename` are required. `tenant` is optional and should be a
+number, indicating which tenant the file or directory should run under. `continuous`
+is optional and should be true/false, indicating that file or directory should be
+run until `pcap-interrupt` is sent or ctrl-c is invoked. `delete-when-done` is
+optional and should be true/false, indicating that the file or files under the
+directory specified by `filename` should be deleted when processing is complete.
+`delete-when-done` defaults to false, indicating files will be kept after
+processing.

python/suricata/sc/suricatasc.py

@@ -184,6 +184,9 @@ class SuricataSC:
                 continuous = None
                 if len(parts) > 4:
                     continuous = parts[4]
+                delete_when_done = None
+                if len(parts) > 5:
+                    delete_when_done = parts[5]
                 if cmd != "pcap-file":
                     raise SuricataCommandException("Invalid command '%s'" % (command))
                 else:
@@ -194,6 +197,8 @@ class SuricataSC:
                         arguments["tenant"] = int(tenant)
                     if continuous != None:
                         arguments["continuous"] = continuous
+                    if delete_when_done != None:
+                        arguments["delete-when-done"] = delete_when_done
             elif "pcap-file-continuous " in command:
                 try:
                     parts = command.split(' ')
@@ -203,6 +208,9 @@ class SuricataSC:
                 tenant = None
                 if len(parts) > 3:
                     tenant = parts[3]
+                delete_when_done = None
+                if len(parts) > 4:
+                    delete_when_done = parts[4]
                 if cmd != "pcap-file":
                     raise SuricataCommandException("Invalid command '%s'" % (command))
                 else:
@@ -212,6 +220,8 @@ class SuricataSC:
                     arguments["continuous"] = True
                     if tenant != None:
                         arguments["tenant"] = int(tenant)
+                    if delete_when_done != None:
+                        arguments["delete-when-done"] = delete_when_done
             elif "iface-stat" in command:
                 try:
                     [cmd, iface] = command.split(' ', 1)

src/runmode-unix-socket.c

@@ -59,6 +59,7 @@ typedef struct PcapFiles_ {
     time_t delay;
     time_t poll_interval;
     bool continuous;
+    bool should_delete;
     TAILQ_ENTRY(PcapFiles_) next;
 } PcapFiles;
 
@@ -238,6 +239,7 @@ static void PcapFilesFree(PcapFiles *cfile)
  * \param output_dir absolute name of directory where log will be put
  * \param tenant_id Id of tenant associated with this file
  * \param continuous If file should be run in continuous mode
+ * \param delete If file should be deleted when done
  * \param delay Delay required for file modified time before being processed
  * \param poll_interval How frequently directory mode polls for new files
  *
@@ -249,6 +251,7 @@ static TmEcode UnixListAddFile(
     const char *output_dir,
     int tenant_id,
     bool continuous,
+    bool should_delete,
     time_t delay,
     time_t poll_interval
 )
@@ -282,6 +285,7 @@ static TmEcode UnixListAddFile(
 
     cfile->tenant_id = tenant_id;
     cfile->continuous = continuous;
+    cfile->should_delete = should_delete;
     cfile->delay = delay;
     cfile->poll_interval = poll_interval;
 
@@ -304,6 +308,7 @@ static TmEcode UnixSocketAddPcapFileImpl(json_t *cmd, json_t* answer, void *data
     const char *filename;
     const char *output_dir;
     int tenant_id = 0;
+    bool should_delete = false;
     time_t delay = 30;
     time_t poll_interval = 5;
 #ifdef OS_WIN32
@@ -368,6 +373,11 @@ static TmEcode UnixSocketAddPcapFileImpl(json_t *cmd, json_t* answer, void *data
         tenant_id = json_number_value(targ);
     }
 
+    json_t *delete_arg = json_object_get(cmd, "delete-when-done");
+    if (delete_arg != NULL) {
+        should_delete = json_is_true(delete_arg);
+    }
+
     json_t *delay_arg = json_object_get(cmd, "delay");
     if (delay_arg != NULL) {
         if (!json_is_integer(delay_arg)) {
@@ -392,7 +402,7 @@ static TmEcode UnixSocketAddPcapFileImpl(json_t *cmd, json_t* answer, void *data
     }
 
     switch (UnixListAddFile(this, filename, output_dir, tenant_id, continuous,
-                           delay, poll_interval)) {
+                           should_delete, delay, poll_interval)) {
         case TM_ECODE_FAILED:
         case TM_ECODE_DONE:
             json_object_set_new(answer, "message",
@@ -502,6 +512,16 @@ static TmEcode UnixSocketPcapFilesCheck(void *data)
         PcapFilesFree(cfile);
         return TM_ECODE_FAILED;
     }
+    if (cfile->should_delete) {
+        set_res = ConfSetFinal("pcap-file.delete-when-done", "true");
+    } else {
+        set_res = ConfSetFinal("pcap-file.delete-when-done", "false");
+    }
+    if (set_res != 1) {
+        SCLogError(SC_ERR_INVALID_ARGUMENT, "Can not set delete mode for pcap processing");
+        PcapFilesFree(cfile);
+        return TM_ECODE_FAILED;
+    }
 
     if (cfile->delay > 0) {
         char tstr[32];

src/source-pcap-file-helper.c

@@ -41,6 +41,13 @@ void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
             pfv->pcap_handle = NULL;
         }
         if (pfv->filename != NULL) {
+            if (pfv->shared->should_delete) {
+                SCLogDebug("Deleting pcap file %s", pfv->filename);
+                if (unlink(pfv->filename) != 0) {
+                    SCLogWarning(SC_ERR_PCAP_FILE_DELETE_FAILED,
+                                 "Failed to delete %s", pfv->filename);
+                }
+            }
             SCFree(pfv->filename);
             pfv->filename = NULL;
         }

src/source-pcap-file-helper.h

@@ -45,6 +45,8 @@ typedef struct PcapFileSharedVars_
 
     struct timespec last_processed;
 
+    bool should_delete;
+
     ThreadVars *tv;
     TmSlot *slot;
 

src/source-pcap-file.c

@@ -231,6 +231,12 @@ TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **d
         }
     }
 
+    int should_delete = 0;
+    ptv->shared.should_delete = false;
+    if (ConfGetBool("pcap-file.delete-when-done", &should_delete) == 1) {
+        ptv->shared.should_delete = should_delete == 1;
+    }
+
     DIR *directory = NULL;
     SCLogInfo("Checking file or directory %s", (char*)initdata);
     if(PcapDetermineDirectoryOrFile((char *)initdata, &directory) == TM_ECODE_FAILED) {

src/suricata.c

@@ -622,6 +622,7 @@ static void PrintUsage(const char *progname)
     printf("\t--build-info                         : display build information\n");
     printf("\t--pcap[=<dev>]                       : run in pcap mode, no value select interfaces from suricata.yaml\n");
     printf("\t--pcap-file-continuous               : when running in pcap mode with a directory, continue checking directory for pcaps until interrupted\n");
+    printf("\t--pcap-file-delete                   : when running in replay mode (-r with directory or file), will delete pcap files that have been processed when done\n");
 #ifdef HAVE_PCAP_SET_BUFF
     printf("\t--pcap-buffer-size                   : size of the pcap buffer value from 0 - %i\n",INT_MAX);
 #endif /* HAVE_SET_PCAP_BUFF */
@@ -1474,6 +1475,7 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri)
         {"netmap", optional_argument, 0, 0},
         {"pcap", optional_argument, 0, 0},
         {"pcap-file-continuous", 0, 0, 0},
+        {"pcap-file-delete", 0, 0, 0},
         {"simulate-ips", 0, 0 , 0},
         {"no-random", 0, &g_disable_randomness, 1},
 
@@ -1887,11 +1889,17 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri)
                 }
             }
             else if (strcmp((long_opts[option_index]).name, "pcap-file-continuous") == 0) {
-                if(ConfSetFinal("pcap-file.continuous", "true") != 1) {
+                if (ConfSetFinal("pcap-file.continuous", "true") != 1) {
                     SCLogError(SC_ERR_CMD_LINE, "Failed to set pcap-file.continuous");
                     return TM_ECODE_FAILED;
                 }
             }
+            else if (strcmp((long_opts[option_index]).name, "pcap-file-delete") == 0) {
+                if (ConfSetFinal("pcap-file.delete-when-done", "true") != 1) {
+                    SCLogError(SC_ERR_CMD_LINE, "Failed to set pcap-file.delete-when-done");
+                    return TM_ECODE_FAILED;
+                }
+            }
             break;
         case 'c':
             suri->conf_filename = optarg;