file: introduce common flags handling function

src/app-layer-htp-file.c

@@ -80,7 +80,7 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
         uint64_t txid, uint8_t direction)
 {
     int retval = 0;
-    uint8_t flags = 0;
+    uint16_t flags = 0;
     FileContainer *files = NULL;
     FileContainer *files_opposite = NULL;
     const StreamingBufferConfig *sbcfg = NULL;
@@ -103,32 +103,13 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
         files = s->files_tc;
         files_opposite = s->files_ts;
 
+        flags = FileFlowToFlags(s->f, STREAM_TOCLIENT);
+
         if ((s->flags & HTP_FLAG_STORE_FILES_TS) ||
                 ((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) {
             flags |= FILE_STORE;
-        }
+            flags &= ~FILE_NOSTORE;
-
+        } else if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TC)) {
-        if (s->f->flags & FLOW_FILE_NO_MAGIC_TC) {
-            SCLogDebug("no magic for this flow in toclient direction, so none for this file");
-            flags |= FILE_NOMAGIC;
-        }
-
-        if (s->f->flags & FLOW_FILE_NO_MD5_TC) {
-            SCLogDebug("no md5 for this flow in toclient direction, so none for this file");
-            flags |= FILE_NOMD5;
-        }
-
-        if (s->f->flags & FLOW_FILE_NO_SHA1_TC) {
-            SCLogDebug("no sha1 for this flow in toclient direction, so none for this file");
-            flags |= FILE_NOSHA1;
-        }
-
-        if (s->f->flags & FLOW_FILE_NO_SHA256_TC) {
-            SCLogDebug("no sha256 for this flow in toclient direction, so none for this file");
-            flags |= FILE_NOSHA256;
-        }
-
-        if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TC)) {
             flags |= FILE_NOSTORE;
         }
 
@@ -146,31 +127,12 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
         files = s->files_ts;
         files_opposite = s->files_tc;
 
+        flags = FileFlowToFlags(s->f, STREAM_TOSERVER);
         if ((s->flags & HTP_FLAG_STORE_FILES_TC) ||
                 ((s->flags & HTP_FLAG_STORE_FILES_TX_TC) && txid == s->store_tx_id)) {
             flags |= FILE_STORE;
-        }
+            flags &= ~FILE_NOSTORE;
-        if (s->f->flags & FLOW_FILE_NO_MAGIC_TS) {
+        } else if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TS)) {
-            SCLogDebug("no magic for this flow in toserver direction, so none for this file");
-            flags |= FILE_NOMAGIC;
-        }
-
-        if (s->f->flags & FLOW_FILE_NO_MD5_TS) {
-            SCLogDebug("no md5 for this flow in toserver direction, so none for this file");
-            flags |= FILE_NOMD5;
-        }
-
-        if (s->f->flags & FLOW_FILE_NO_SHA1_TS) {
-            SCLogDebug("no sha1 for this flow in toserver direction, so none for this file");
-            flags |= FILE_NOSHA1;
-        }
-
-        if (s->f->flags & FLOW_FILE_NO_SHA256_TS) {
-            SCLogDebug("no sha256 for this flow in toserver direction, so none for this file");
-            flags |= FILE_NOSHA256;
-        }
-
-        if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TS)) {
             flags |= FILE_NOSTORE;
         }
 

src/app-layer-smtp.c

@@ -408,28 +408,10 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
     SMTPState *smtp_state = (SMTPState *) flow->alstate;
     MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
     FileContainer *files = NULL;
-    uint16_t flags = 0;
 
-    /* Set flags */
+    uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER);
-    if (flow->flags & FLOW_FILE_NO_STORE_TS) {
+    /* we depend on detection engine for file pruning */
-        flags |= FILE_NOSTORE;
+    flags |= FILE_USE_DETECT;
-    }
-
-    if (flow->flags & FLOW_FILE_NO_MAGIC_TS) {
-        flags |= FILE_NOMAGIC;
-    }
-
-    if (flow->flags & FLOW_FILE_NO_MD5_TS) {
-        flags |= FILE_NOMD5;
-    }
-
-    if (flow->flags & FLOW_FILE_NO_SHA1_TS) {
-        flags |= FILE_NOSHA1;
-    }
-
-    if (flow->flags & FLOW_FILE_NO_SHA256_TS) {
-        flags |= FILE_NOSHA256;
-    }
 
     /* Find file */
     if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
@@ -464,7 +446,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
             }
 
             if (FileOpenFile(files, &smtp_config.sbcfg, (uint8_t *) entity->filename, entity->filename_len,
-                    (uint8_t *) chunk, len, flags|FILE_USE_DETECT) == NULL) {
+                    (uint8_t *) chunk, len, flags) == NULL) {
                 ret = MIME_DEC_ERR_DATA;
                 SCLogDebug("FileOpenFile() failed");
             }

src/util-file.c

@@ -175,6 +175,54 @@ void FileForceHashParseCfg(ConfNode *conf)
     }
 }
 
+uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
+{
+    uint16_t flags = 0;
+
+    if (direction == STREAM_TOSERVER) {
+        if (flow->flags & FLOW_FILE_NO_STORE_TS) {
+            flags |= FILE_NOSTORE;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_MAGIC_TS) {
+            flags |= FILE_NOMAGIC;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_MD5_TS) {
+            flags |= FILE_NOMD5;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_SHA1_TS) {
+            flags |= FILE_NOSHA1;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_SHA256_TS) {
+            flags |= FILE_NOSHA256;
+        }
+    } else {
+        if (flow->flags & FLOW_FILE_NO_STORE_TC) {
+            flags |= FILE_NOSTORE;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_MAGIC_TC) {
+            flags |= FILE_NOMAGIC;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_MD5_TC) {
+            flags |= FILE_NOMD5;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_SHA1_TC) {
+            flags |= FILE_NOSHA1;
+        }
+
+        if (flow->flags & FLOW_FILE_NO_SHA256_TC) {
+            flags |= FILE_NOSHA256;
+        }
+    }
+    return flags;
+}
+
 int FileMagicSize(void)
 {
     /** \todo make this size configurable */

src/util-file.h

@@ -33,20 +33,20 @@
 
 #include "util-streaming-buffer.h"
 
-#define FILE_TRUNCATED  0x0001
+#define FILE_TRUNCATED  BIT_U16(0)
-#define FILE_NOMAGIC    0x0002
+#define FILE_NOMAGIC    BIT_U16(1)
-#define FILE_NOMD5      0x0004
+#define FILE_NOMD5      BIT_U16(2)
-#define FILE_MD5        0x0008
+#define FILE_MD5        BIT_U16(3)
-#define FILE_NOSHA1     0x0010
+#define FILE_NOSHA1     BIT_U16(4)
-#define FILE_SHA1       0x0020
+#define FILE_SHA1       BIT_U16(5)
-#define FILE_NOSHA256   0x0040
+#define FILE_NOSHA256   BIT_U16(6)
-#define FILE_SHA256     0x0080
+#define FILE_SHA256     BIT_U16(7)
-#define FILE_LOGGED     0x0100
+#define FILE_LOGGED     BIT_U16(8)
-#define FILE_NOSTORE    0x0200
+#define FILE_NOSTORE    BIT_U16(9)
-#define FILE_STORE      0x0400
+#define FILE_STORE      BIT_U16(10)
-#define FILE_STORED     0x0800
+#define FILE_STORED     BIT_U16(11)
-#define FILE_NOTRACK    0x1000 /**< track size of file */
+#define FILE_NOTRACK    BIT_U16(12) /**< track size of file */
-#define FILE_USE_DETECT 0x2000 /**< use content_inspected tracker */
+#define FILE_USE_DETECT BIT_U16(13) /**< use content_inspected tracker */
 
 typedef enum FileState_ {
     FILE_STATE_NONE = 0,    /**< no state */
@@ -211,4 +211,6 @@ void FileTruncateAllOpenFiles(FileContainer *);
 
 uint64_t FileSize(const File *file);
 
+uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction);
+
 #endif /* __UTIL_FILE_H__ */