file-store: add depth setting

When a rules match and fired filestore we may want
to increase the stream reassembly depth for this specific.

This add the 'depth' setting in file-store config,
which permits to specify how much data we want to reassemble
into a stream.

doc/userguide/file-extraction/file-extraction.rst

@@ -16,7 +16,7 @@ Settings
 
 *stream.checksum_validation* controls whether or not the stream engine rejects packets with invalid checksums. A good idea normally, but the network interface performs checksum offloading a lot of packets may seem to be broken. This setting is enabled by default, and can be disabled by setting to "no". Note that the checksum handling can be controlled per interface, see "checksum_checks" in example configuration.
 
-*stream.reassembly.depth* controls how far into a stream reassembly is done. Beyond this value no reassembly will be done. This means that after this value the HTTP session will no longer be tracked. By default a settings of 1 Megabyte is used. 0 sets it to unlimited.
+*file-store.stream-depth* controls how far into a stream reassembly is done. Beyond this value no reassembly will be done. This means that after this value the HTTP session will no longer be tracked. By default a settings of 1 Megabyte is used. 0 sets it to unlimited. If set to no, it is disabled and stream.reassembly.depth is considered.
 
 *libhtp.default-config.request-body-limit* / *libhtp.server-config.<config>.request-body-limit* controls how much of the HTTP request body is tracked for inspection by the http_client_body keyword, but also used to limit file inspection. A value of 0 means unlimited.
 
@@ -40,6 +40,7 @@ drop dir must be configured.
       log-dir: files    # directory to store the files
       force-magic: no   # force logging magic on all stored files
       force-md5: no     # force logging of md5 checksums
+      stream-depth: 1mb # reassemble 1mb into a stream, set to no to disable
       waldo: file.waldo # waldo file to store the file_id across runs
 
 Each file that is stored with have a name "file.<id>". The id will be reset and files will be overwritten unless the waldo option is used.

src/detect-filestore.c

@@ -198,6 +198,10 @@ int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Pack
 #endif
     }
 
+    /* set filestore depth for stream reassembling */
+    TcpSession *ssn = (TcpSession *)p->flow->protoctx;
+    TcpSessionSetReassemblyDepth(ssn, FileReassemblyDepth());
+
     if (p->flowflags & FLOW_PKT_TOCLIENT)
         flags |= STREAM_TOCLIENT;
     else

src/log-filestore.c

@@ -46,6 +46,7 @@
 #include "util-atomic.h"
 #include "util-file.h"
 #include "util-time.h"
+#include "util-misc.h"
 
 #include "output.h"
 
@@ -484,6 +485,21 @@ static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf)
     FileForceHashParseCfg(conf);
     SCLogInfo("storing files in %s", g_logfile_base_dir);
 
+    const char *stream_depth_str = ConfNodeLookupChildValue(conf, "stream-depth");
+    if (stream_depth_str != NULL && strcmp(stream_depth_str, "no")) {
+        uint32_t stream_depth = 0;
+        if (ParseSizeStringU32(stream_depth_str,
+                               &stream_depth) < 0) {
+            SCLogError(SC_ERR_SIZE_PARSE, "Error parsing "
+                       "file-store.stream-depth "
+                       "from conf file - %s.  Killing engine",
+                       stream_depth_str);
+            exit(EXIT_FAILURE);
+        } else {
+            FileReassemblyDepthEnable(stream_depth);
+        }
+    }
+
     SCReturnPtr(output_ctx, "OutputCtx");
 }
 

src/util-file.c

@@ -28,6 +28,7 @@
 #include "debug.h"
 #include "flow.h"
 #include "stream.h"
+#include "stream-tcp.h"
 #include "runmodes.h"
 #include "util-hash.h"
 #include "util-debug.h"
@@ -66,6 +67,16 @@ static int g_file_force_sha256 = 0;
  */
 static int g_file_force_tracking = 0;
 
+/** \brief switch to use g_file_store_reassembly_depth
+ *         to reassembly files
+ */
+static int g_file_store_enable = 0;
+
+/** \brief stream_config.reassembly_depth equivalent
+ *         for files
+ */
+static uint32_t g_file_store_reassembly_depth = 0;
+
 /* prototypes */
 static void FileFree(File *);
 
@@ -99,6 +110,20 @@ int FileForceFilestore(void)
     return g_file_force_filestore;
 }
 
+void FileReassemblyDepthEnable(uint32_t size)
+{
+    g_file_store_enable = 1;
+    g_file_store_reassembly_depth = size;
+}
+
+uint32_t FileReassemblyDepth(void)
+{
+    if (g_file_store_enable == 1)
+        return g_file_store_reassembly_depth;
+    else
+        return stream_config.reassembly_depth;
+}
+
 int FileForceMagic(void)
 {
     return g_file_force_magic;

src/util-file.h

@@ -182,6 +182,8 @@ void FilePrune(FileContainer *ffc);
 
 void FileForceFilestoreEnable(void);
 int FileForceFilestore(void);
+void FileReassemblyDepthEnable(uint32_t size);
+uint32_t FileReassemblyDepth(void);
 
 void FileDisableMagic(Flow *f, uint8_t);
 void FileForceMagicEnable(void);

suricata.yaml.in

@@ -394,7 +394,7 @@ outputs:
   # file "file.<id>.meta" is created.
   #
   # File extraction depends on a lot of things to be fully done:
-  # - stream reassembly depth. For optimal results, set this to 0 (unlimited)
+  # - file-store stream-depth. For optimal results, set this to 0 (unlimited)
   # - http request / response body sizes. Again set to 0 for optimal results.
   # - rules that contain the "filestore" keyword.
   - file-store:
@@ -405,6 +405,7 @@ outputs:
       # sha1 and sha256
       #force-hash: [md5]
       force-filestore: no # force storing of all files
+      stream-depth: 1mb # reassemble 1mb into a stream, set to no to disable
       #waldo: file.waldo # waldo file to store the file_id across runs
 
   # output module to log files tracked in a easily parsable json format