file-hashing: added support for SHA-1 file hashing

src/Makefile.am

@@ -138,6 +138,7 @@ detect-file-data.c detect-file-data.h \
 detect-fileext.c detect-fileext.h \
 detect-filemagic.c detect-filemagic.h \
 detect-filemd5.c detect-filemd5.h \
+detect-filesha1.c detect-filesha1.h \
 detect-filename.c detect-filename.h \
 detect-filesize.c detect-filesize.h \
 detect-filestore.c detect-filestore.h \

src/app-layer-htp-file.c

@@ -118,6 +118,11 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
             flags |= FILE_NOMD5;
         }
 
+        if (s->f->flags & FLOW_FILE_NO_SHA1_TC) {
+            SCLogDebug("no sha1 for this flow in toclient direction, so none for this file");
+            flags |= FILE_NOSHA1;
+        }
+
         if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TC)) {
             flags |= FILE_NOSTORE;
         }
@@ -150,6 +155,11 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
             flags |= FILE_NOMD5;
         }
 
+        if (s->f->flags & FLOW_FILE_NO_SHA1_TS) {
+            SCLogDebug("no sha1 for this flow in toserver direction, so none for this file");
+            flags |= FILE_NOSHA1;
+        }
+
         if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TS)) {
             flags |= FILE_NOSTORE;
         }

src/app-layer-smtp.c

@@ -423,6 +423,10 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
         flags |= FILE_NOMD5;
     }
 
+    if (flow->flags & FLOW_FILE_NO_SHA1_TS) {
+        flags |= FILE_NOSHA1;
+    }
+
     /* Find file */
     if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
 

src/detect-engine-file.c

@@ -129,6 +129,12 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
                 break;
             }
 
+            if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) {
+                SCLogDebug("sig needs file sha1, but we don't have any");
+                r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+                break;
+            }
+
             if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) {
                 SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED");
                 r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

src/detect-engine-siggroup.c

@@ -516,7 +516,7 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 }
 
 /**
- *  \brief Set the need md5 flag in the sgh.
+ *  \brief Set the need magic flag in the sgh.
  *
  *  \param de_ctx detection engine ctx for the signatures
  *  \param sgh sig group head to set the flag in
@@ -608,12 +608,12 @@ void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 }
 
 /**
- *  \brief Set the need magic flag in the sgh.
+ *  \brief Set the need hash flag in the sgh.
  *
  *  \param de_ctx detection engine ctx for the signatures
  *  \param sgh sig group head to set the flag in
  */
-void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 {
     Signature *s = NULL;
     uint32_t sig = 0;
@@ -631,6 +631,12 @@ void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             SCLogDebug("sgh %p has filemd5", sgh);
             break;
         }
+
+        if (SignatureIsFileSha1Inspecting(s)) {
+            sgh->flags |= SIG_GROUP_HEAD_HAVEFILESHA1;
+            SCLogDebug("sgh %p has filesha1", sgh);
+            break;
+        }
     }
 
     return;

src/detect-engine-siggroup.h

@@ -67,7 +67,7 @@ void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 void SigGroupHeadStore(DetectEngineCtx *, SigGroupHead *);
 void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *, SigGroupHead *);
 void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *);
-void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *, SigGroupHead *);
+void SigGroupHeadSetFileHashFlag(DetectEngineCtx *, SigGroupHead *);
 void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *, SigGroupHead *);
 uint16_t SigGroupHeadGetMinMpmSize(DetectEngineCtx *de_ctx,
                                    SigGroupHead *sgh, int list);

src/detect-filesha1.c

@@ -0,0 +1,162 @@
+/* Copyright (C) 2007-2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ * \author Duarte Silva <duarte.silva@serializing.me>
+ *
+ */
+
+#include "suricata-common.h"
+
+#include "util-detect-file-hash.h"
+
+#include "util-unittest.h"
+#include "util-unittest-helper.h"
+
+#include "detect-filesha1.h"
+
+#ifndef HAVE_NSS
+
+static int DetectFileSha1SetupNoSupport (DetectEngineCtx *a, Signature *b, char *c)
+{
+    SCLogError(SC_ERR_NO_SHA1_SUPPORT, "no SHA-1 calculation support built in, needed for filesha1 keyword");
+    return -1;
+}
+
+/**
+ * \brief Registration function for keyword: filesha1
+ */
+void DetectFileSha1Register(void)
+{
+    sigmatch_table[DETECT_FILESHA1].name = "filesha1";
+    sigmatch_table[DETECT_FILESHA1].FileMatch = NULL;
+    sigmatch_table[DETECT_FILESHA1].Setup = DetectFileSha1SetupNoSupport;
+    sigmatch_table[DETECT_FILESHA1].Free  = NULL;
+    sigmatch_table[DETECT_FILESHA1].RegisterTests = NULL;
+    sigmatch_table[DETECT_FILESHA1].flags = SIGMATCH_NOT_BUILT;
+
+    SCLogDebug("registering filesha1 rule option");
+    return;
+}
+
+#else /* HAVE_NSS */
+
+static int DetectFileSha1Setup (DetectEngineCtx *, Signature *, char *);
+static void DetectFileSha1RegisterTests(void);
+
+/**
+ * \brief Registration function for keyword: filesha1
+ */
+void DetectFileSha1Register(void)
+{
+    sigmatch_table[DETECT_FILESHA1].name = "filesha1";
+    sigmatch_table[DETECT_FILESHA1].desc = "match file SHA-1 against list of SHA-1 checksums";
+    sigmatch_table[DETECT_FILESHA1].url = "https://redmine.openinfosecfoundation.org/projects/suricata/wiki/File-keywords#filesha1";
+    sigmatch_table[DETECT_FILESHA1].FileMatch = DetectFileHashMatch;
+    sigmatch_table[DETECT_FILESHA1].Setup = DetectFileSha1Setup;
+    sigmatch_table[DETECT_FILESHA1].Free  = DetectFileHashFree;
+    sigmatch_table[DETECT_FILESHA1].RegisterTests = DetectFileSha1RegisterTests;
+
+    SCLogDebug("registering filesha1 rule option");
+    return;
+}
+
+/**
+ * \brief this function is used to parse filesha1 options
+ * \brief into the current signature
+ *
+ * \param de_ctx pointer to the Detection Engine Context
+ * \param s pointer to the Current Signature
+ * \param str pointer to the user provided "filesha1" option
+ *
+ * \retval 0 on Success
+ * \retval -1 on Failure
+ */
+static int DetectFileSha1Setup (DetectEngineCtx *de_ctx, Signature *s, char *str)
+{
+    return DetectFileHashSetup(de_ctx, s, str, DETECT_FILESHA1);
+}
+
+#ifdef UNITTESTS
+static int SHA1MatchLookupString(ROHashTable *hash, char *string)
+{
+    uint8_t sha1[20];
+    if (ReadHashString(sha1, string, "file", 88, 40) == 1) {
+        void *ptr = ROHashLookup(hash, &sha1, (uint16_t)sizeof(sha1));
+        if (ptr == NULL)
+            return 0;
+        else
+            return 1;
+    }
+    return 0;
+}
+
+static int SHA1MatchTest01(void)
+{
+    ROHashTable *hash = ROHashInit(4, 20);
+    if (hash == NULL) {
+        return 0;
+    }
+    if (LoadHashTable(hash, "447661c5de965bd4d837b50244467e37bddc184d", "file", 1, DETECT_FILESHA1) != 1)
+        return 0;
+    if (LoadHashTable(hash, "75a9af1e34dc0bb2f7fcde9d56b2503072ac35dd", "file", 2, DETECT_FILESHA1) != 1)
+        return 0;
+    if (LoadHashTable(hash, "53224a297bbb30631670fdcd2d295d87a1d328e9", "file", 3, DETECT_FILESHA1) != 1)
+        return 0;
+    if (LoadHashTable(hash, "3395856ce81f2b7382dee72602f798b642f14140", "file", 4, DETECT_FILESHA1) != 1)
+        return 0;
+    if (LoadHashTable(hash, "65559245709fe98052eb284577f1fd61c01ad20d", "file", 5, DETECT_FILESHA1) != 1)
+        return 0;
+    if (LoadHashTable(hash, "0931fd4e05e6ea81c75f8488ecc1db9e66f22cbb", "file", 6, DETECT_FILESHA1) != 1)
+        return 0;
+
+    if (ROHashInitFinalize(hash) != 1) {
+        return 0;
+    }
+
+    if (SHA1MatchLookupString(hash, "447661c5de965bd4d837b50244467e37bddc184d") != 1)
+        return 0;
+    if (SHA1MatchLookupString(hash, "75a9af1e34dc0bb2f7fcde9d56b2503072ac35dd") != 1)
+        return 0;
+    if (SHA1MatchLookupString(hash, "53224a297bbb30631670fdcd2d295d87a1d328e9") != 1)
+        return 0;
+    if (SHA1MatchLookupString(hash, "3395856ce81f2b7382dee72602f798b642f14140") != 1)
+        return 0;
+    if (SHA1MatchLookupString(hash, "65559245709fe98052eb284577f1fd61c01ad20d") != 1)
+        return 0;
+    if (SHA1MatchLookupString(hash, "0931fd4e05e6ea81c75f8488ecc1db9e66f22cbb") != 1)
+        return 0;
+    /* Shouldn't match */
+    if (SHA1MatchLookupString(hash, "3333333333333333333333333333333333333333") == 1)
+        return 0;
+
+    ROHashFree(hash);
+    return 1;
+}
+#endif
+
+void DetectFileSha1RegisterTests(void)
+{
+#ifdef UNITTESTS
+    UtRegisterTest("SHA1MatchTest01", SHA1MatchTest01);
+#endif
+}
+
+#endif /* HAVE_NSS */

src/detect-filesha1.h

@@ -0,0 +1,31 @@
+/* Copyright (C) 2007-2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ * \author Duarte Silva <duarte.silva@serializing.me>
+ */
+
+#ifndef __DETECT_FILESHA1_H__
+#define __DETECT_FILESHA1_H__
+
+/* prototypes */
+void DetectFileSha1Register (void);
+
+#endif /* __DETECT_FILESHA1_H__ */

src/detect.c

@@ -108,6 +108,7 @@
 #include "detect-filestore.h"
 #include "detect-filemagic.h"
 #include "detect-filemd5.h"
+#include "detect-filesha1.h"
 #include "detect-filesize.h"
 #include "detect-dsize.h"
 #include "detect-flowvar.h"
@@ -1861,6 +1862,14 @@ end:
                     FileDisableMd5(pflow, STREAM_TOSERVER);
                 }
 
+                /* see if this sgh requires us to consider file sha1 */
+                if (!FileForceSha1() && (pflow->sgh_toserver == NULL ||
+                            !(pflow->sgh_toserver->flags & SIG_GROUP_HEAD_HAVEFILESHA1)))
+                {
+                    SCLogDebug("disabling sha1 for flow");
+                    FileDisableSha1(pflow, STREAM_TOSERVER);
+                }
+
                 /* see if this sgh requires us to consider filesize */
                 if (pflow->sgh_toserver == NULL ||
                             !(pflow->sgh_toserver->flags & SIG_GROUP_HEAD_HAVEFILESIZE))
@@ -1892,6 +1901,14 @@ end:
                     FileDisableMd5(pflow, STREAM_TOCLIENT);
                 }
 
+                /* check if this flow needs sha1, if not disable it */
+                if (!FileForceSha1() && (pflow->sgh_toclient == NULL ||
+                            !(pflow->sgh_toclient->flags & SIG_GROUP_HEAD_HAVEFILESHA1)))
+                {
+                    SCLogDebug("disabling sha1 for flow");
+                    FileDisableSha1(pflow, STREAM_TOCLIENT);
+                }
+
                 /* see if this sgh requires us to consider filesize */
                 if (pflow->sgh_toclient == NULL ||
                             !(pflow->sgh_toclient->flags & SIG_GROUP_HEAD_HAVEFILESIZE))
@@ -2146,10 +2163,23 @@ int SignatureIsFilemagicInspecting(Signature *s)
  */
 int SignatureIsFileMd5Inspecting(Signature *s)
 {
-    if (s == NULL)
-        return 0;
+    if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
+        return 1;
+
+    return 0;
+}
 
-    if (s->file_flags & FILE_SIG_NEED_MD5)
+/**
+ *  \brief Check if a signature contains the filesha1 keyword.
+ *
+ *  \param s signature
+ *
+ *  \retval 0 no
+ *  \retval 1 yes
+ */
+int SignatureIsFileSha1Inspecting(Signature *s)
+{
+    if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
         return 1;
 
     return 0;
@@ -3996,7 +4026,7 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
         SCLogDebug("sgh %p", sgh);
 
         SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
-        SigGroupHeadSetFileMd5Flag(de_ctx, sgh);
+        SigGroupHeadSetFileHashFlag(de_ctx, sgh);
         SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
         SigGroupHeadSetFilestoreCount(de_ctx, sgh);
         SCLogDebug("filestore count %u", sgh->filestore_cnt);
@@ -4378,6 +4408,7 @@ void SigTableSetup(void)
     DetectFilestoreRegister();
     DetectFilemagicRegister();
     DetectFileMd5Register();
+    DetectFileSha1Register();
     DetectFilesizeRegister();
     DetectAppLayerEventRegister();
     DetectHttpUARegister();

src/detect.h

@@ -937,11 +937,13 @@ typedef struct SigTableElmt_ {
 #define SIG_GROUP_HEAD_HAVEFILEMAGIC    (1 << 20)
 #define SIG_GROUP_HEAD_HAVEFILEMD5      (1 << 21)
 #define SIG_GROUP_HEAD_HAVEFILESIZE     (1 << 22)
-#define SIG_GROUP_HEAD_MPM_DNSQUERY     (1 << 23)
-#define SIG_GROUP_HEAD_MPM_TLSSNI       (1 << 24)
-#define SIG_GROUP_HEAD_MPM_TLSISSUER    (1 << 25)
-#define SIG_GROUP_HEAD_MPM_TLSSUBJECT   (1 << 26)
-#define SIG_GROUP_HEAD_MPM_FD_SMTP      (1 << 27)
+#define SIG_GROUP_HEAD_HAVEFILESHA1     (1 << 23)
+
+#define SIG_GROUP_HEAD_MPM_DNSQUERY     (1 << 24)
+#define SIG_GROUP_HEAD_MPM_TLSSNI       (1 << 25)
+#define SIG_GROUP_HEAD_MPM_TLSISSUER    (1 << 26)
+#define SIG_GROUP_HEAD_MPM_TLSSUBJECT   (1 << 27)
+#define SIG_GROUP_HEAD_MPM_FD_SMTP      (1 << 28)
 
 #define APP_MPMS_MAX 21
 
@@ -1278,6 +1280,7 @@ Signature *DetectGetTagSignature(void);
 int SignatureIsFilestoring(Signature *);
 int SignatureIsFilemagicInspecting(Signature *);
 int SignatureIsFileMd5Inspecting(Signature *);
+int SignatureIsFileSha1Inspecting(Signature *s);
 int SignatureIsFilesizeInspecting(Signature *);
 
 int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int);

src/flow.h

@@ -66,40 +66,44 @@ typedef struct AppLayerParserState_ AppLayerParserState;
 #define FLOW_ACTION_DROP                  0x00000200
 
 /** Sgh for toserver direction set (even if it's NULL) */
-#define FLOW_SGH_TOSERVER                 0x00000800
+#define FLOW_SGH_TOSERVER                 0x00000400
 /** Sgh for toclient direction set (even if it's NULL) */
-#define FLOW_SGH_TOCLIENT                 0x00001000
+#define FLOW_SGH_TOCLIENT                 0x00000800
 
 /** packet to server direction has been logged in drop file (only in IPS mode) */
-#define FLOW_TOSERVER_DROP_LOGGED         0x00002000
+#define FLOW_TOSERVER_DROP_LOGGED         0x00001000
 /** packet to client direction has been logged in drop file (only in IPS mode) */
-#define FLOW_TOCLIENT_DROP_LOGGED         0x00004000
+#define FLOW_TOCLIENT_DROP_LOGGED         0x00002000
 /** alproto detect done.  Right now we need it only for udp */
-#define FLOW_ALPROTO_DETECT_DONE          0x00008000
+#define FLOW_ALPROTO_DETECT_DONE          0x00004000
 
 // vacany 1x
 
 /** Pattern matcher alproto detection done */
-#define FLOW_TS_PM_ALPROTO_DETECT_DONE    0x00020000
+#define FLOW_TS_PM_ALPROTO_DETECT_DONE    0x00008000
 /** Probing parser alproto detection done */
-#define FLOW_TS_PP_ALPROTO_DETECT_DONE    0x00040000
+#define FLOW_TS_PP_ALPROTO_DETECT_DONE    0x00010000
 /** Pattern matcher alproto detection done */
-#define FLOW_TC_PM_ALPROTO_DETECT_DONE    0x00100000
+#define FLOW_TC_PM_ALPROTO_DETECT_DONE    0x00020000
 /** Probing parser alproto detection done */
-#define FLOW_TC_PP_ALPROTO_DETECT_DONE    0x00200000
-#define FLOW_TIMEOUT_REASSEMBLY_DONE      0x00800000
+#define FLOW_TC_PP_ALPROTO_DETECT_DONE    0x00040000
+#define FLOW_TIMEOUT_REASSEMBLY_DONE      0x00080000
 /** even if the flow has files, don't store 'm */
-#define FLOW_FILE_NO_STORE_TS             0x01000000
-#define FLOW_FILE_NO_STORE_TC             0x02000000
+#define FLOW_FILE_NO_STORE_TS             0x00100000
+#define FLOW_FILE_NO_STORE_TC             0x00200000
 
 /** flow is ipv4 */
-#define FLOW_IPV4                         0x04000000
+#define FLOW_IPV4                         0x00400000
 /** flow is ipv6 */
-#define FLOW_IPV6                         0x08000000
+#define FLOW_IPV6                         0x00800000
 
 /** no md5 on files in this flow */
-#define FLOW_FILE_NO_MD5_TS               0x10000000
-#define FLOW_FILE_NO_MD5_TC               0x20000000
+#define FLOW_FILE_NO_MD5_TS               0x01000000
+#define FLOW_FILE_NO_MD5_TC               0x02000000
+
+/** no sha1 on files in this flow */
+#define FLOW_FILE_NO_SHA1_TS              0x04000000
+#define FLOW_FILE_NO_SHA1_TC              0x08000000
 
 /** no size tracking of files in this flow */
 #define FLOW_FILE_NO_SIZE_TS              0x40000000

src/log-file.c

@@ -290,6 +290,14 @@ static void LogFileWriteJsonRecord(LogFileLogThread *aft, const Packet *p, const
                 }
                 fprintf(fp, "\", ");
             }
+            if (ff->flags & FILE_SHA1) {
+                fprintf(fp, "\"sha1\": \"");
+                size_t x;
+                for (x = 0; x < sizeof(ff->sha1); x++) {
+                    fprintf(fp, "%02x", ff->sha1[x]);
+                }
+                fprintf(fp, "\", ");
+            }
 #endif
             break;
         case FILE_STATE_TRUNCATED:
@@ -437,7 +445,6 @@ static OutputCtx *LogFileLogInitCtx(ConfNode *conf)
         SCLogInfo("md5 calculation requires linking against libnss");
 #endif
     }
-
     FileForceTrackingEnable();
     SCReturnPtr(output_ctx, "OutputCtx");
 }

src/log-filestore.c

@@ -261,6 +261,14 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
                     }
                     fprintf(fp, "\n");
                 }
+                if (ff->flags & FILE_SHA1) {
+                    fprintf(fp, "SHA1:              ");
+                    size_t x;
+                    for (x = 0; x < sizeof(ff->sha1); x++) {
+                        fprintf(fp, "%02x", ff->sha1[x]);
+                    }
+                    fprintf(fp, "\n");
+                }
 #endif
                 break;
             case FILE_STATE_TRUNCATED:

src/output-json-file.c

@@ -134,6 +134,15 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
                 }
                 json_object_set_new(fjs, "md5", json_string(s));
             }
+            if (ff->flags & FILE_SHA1) {
+                size_t x;
+                int i;
+                char s[256];
+                for (i = 0, x = 0; x < sizeof(ff->sha1); x++) {
+                    i += snprintf(&s[i], 255-i, "%02x", ff->sha1[x]);
+                }
+                json_object_set_new(fjs, "sha1", json_string(s));
+            }
 #endif
             break;
         case FILE_STATE_TRUNCATED:

src/suricata.c

@@ -2332,7 +2332,7 @@ static int PostConfLoadedSetup(SCInstance *suri)
 
 #ifdef HAVE_NSS
     if (suri->run_mode != RUNMODE_CONF_TEST) {
-        /* init NSS for md5 */
+        /* init NSS for hashing */
         PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
         NSS_NoDB_Init(NULL);
     }

src/util-error.c

@@ -327,6 +327,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_NIC_OFFLOADING);
         CASE_CODE (SC_ERR_NO_FILES_FOR_PROTOCOL);
         CASE_CODE (SC_ERR_INVALID_HASH);
+        CASE_CODE (SC_ERR_NO_SHA1_SUPPORT);
     }
 
     return "UNKNOWN_ERROR";

src/util-error.h

@@ -317,6 +317,7 @@ typedef enum {
     SC_ERR_NIC_OFFLOADING,
     SC_ERR_NO_FILES_FOR_PROTOCOL,
     SC_ERR_INVALID_HASH,
+    SC_ERR_NO_SHA1_SUPPORT,
 } SCError;
 
 const char *SCErrorToString(SCError);

src/util-file.c

@@ -51,6 +51,11 @@ static int g_file_force_magic = 0;
  */
 static int g_file_force_md5 = 0;
 
+/** \brief switch to force sha1 calculation on all files
+ *         regardless of the rules.
+ */
+static int g_file_force_sha1 = 0;
+
 /** \brief switch to force tracking off all files
  *         regardless of the rules.
  */
@@ -74,6 +79,11 @@ void FileForceMd5Enable(void)
     g_file_force_md5 = 1;
 }
 
+void FileForceSha1Enable(void)
+{
+    g_file_force_sha1 = 1;
+}
+
 int FileForceFilestore(void)
 {
     return g_file_force_filestore;
@@ -89,6 +99,11 @@ int FileForceMd5(void)
     return g_file_force_md5;
 }
 
+int FileForceSha1(void)
+{
+    return g_file_force_sha1;
+}
+
 void FileForceTrackingEnable(void)
 {
     g_file_force_tracking = 1;
@@ -289,6 +304,8 @@ static void FileFree(File *ff)
 #ifdef HAVE_NSS
     if (ff->md5_ctx)
         HASH_Destroy(ff->md5_ctx);
+    if (ff->sha1_ctx)
+        HASH_Destroy(ff->sha1_ctx);
 #endif
     SCFree(ff);
 }
@@ -363,6 +380,9 @@ static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
     if (file->md5_ctx) {
         HASH_Update(file->md5_ctx, data, data_len);
     }
+    if (file->sha1_ctx) {
+        HASH_Update(file->sha1_ctx, data, data_len);
+    }
 #endif
     SCReturnInt(0);
 }
@@ -396,11 +416,13 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
 
     if (FileStoreNoStoreCheck(ffc->tail) == 1) {
 #ifdef HAVE_NSS
-        /* no storage but forced md5 */
+        /* no storage but forced hashing */
         if (ffc->tail->md5_ctx) {
-            if (ffc->tail->md5_ctx)
-                HASH_Update(ffc->tail->md5_ctx, data, data_len);
-
+            HASH_Update(ffc->tail->md5_ctx, data, data_len);
+            SCReturnInt(0);
+        }
+        if (ffc->tail->sha1_ctx) {
+            HASH_Update(ffc->tail->sha1_ctx, data, data_len);
             SCReturnInt(0);
         }
 #endif
@@ -471,6 +493,10 @@ File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
         SCLogDebug("not doing md5 for this file");
         ff->flags |= FILE_NOMD5;
     }
+    if (flags & FILE_NOSHA1) {
+        SCLogDebug("not doing sha1 for this file");
+        ff->flags |= FILE_NOSHA1;
+    }
     if (flags & FILE_USE_DETECT) {
         SCLogDebug("considering content_inspect tracker when pruning");
         ff->flags |= FILE_USE_DETECT;
@@ -483,6 +509,12 @@ File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
             HASH_Begin(ff->md5_ctx);
         }
     }
+    if (!(ff->flags & FILE_NOSHA1) || g_file_force_sha1) {
+        ff->sha1_ctx = HASH_Create(HASH_AlgSHA1);
+        if (ff->sha1_ctx != NULL) {
+            HASH_Begin(ff->sha1_ctx);
+        }
+    }
 #endif
 
     ff->state = FILE_STATE_OPENED;
@@ -517,9 +549,11 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
     if (data != NULL) {
         if (ff->flags & FILE_NOSTORE) {
 #ifdef HAVE_NSS
-            /* no storage but md5 */
+            /* no storage but hashing */
             if (ff->md5_ctx)
                 HASH_Update(ff->md5_ctx, data, data_len);
+            if (ff->sha1_ctx)
+                HASH_Update(ff->sha1_ctx, data, data_len);
 #endif
         } else {
             if (AppendData(ff, data, data_len) != 0) {
@@ -547,6 +581,11 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
             HASH_End(ff->md5_ctx, ff->md5, &len, sizeof(ff->md5));
             ff->flags |= FILE_MD5;
         }
+        if (ff->sha1_ctx) {
+            unsigned int len = 0;
+            HASH_End(ff->sha1_ctx, ff->sha1, &len, sizeof(ff->sha1));
+            ff->flags |= FILE_SHA1;
+        }
 #endif
     }
 
@@ -682,6 +721,45 @@ void FileDisableMd5(Flow *f, uint8_t direction)
     SCReturn;
 }
 
+/**
+ *  \brief disable file sha1 calc for this flow
+ *
+ *  \param f *LOCKED* flow
+ *  \param direction flow direction
+*/
+void FileDisableSha1(Flow *f, uint8_t direction)
+{
+    File *ptr = NULL;
+
+    SCEnter();
+
+    DEBUG_ASSERT_FLOW_LOCKED(f);
+
+    if (direction == STREAM_TOSERVER)
+        f->flags |= FLOW_FILE_NO_SHA1_TS;
+    else
+        f->flags |= FLOW_FILE_NO_SHA1_TC;
+
+    FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
+    if (ffc != NULL) {
+        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
+            SCLogDebug("disabling sha1 for file %p from direction %s",
+                    ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
+            ptr->flags |= FILE_NOSHA1;
+
+#ifdef HAVE_NSS
+            /* destroy any ctx we may have so far */
+            if (ptr->sha1_ctx != NULL) {
+                HASH_Destroy(ptr->sha1_ctx);
+                ptr->sha1_ctx = NULL;
+            }
+#endif
+        }
+    }
+
+    SCReturn;
+}
+
 /**
  *  \brief disable file size tracking for this flow
  *
@@ -731,7 +809,7 @@ void FileDisableStoringForFile(File *ff)
     ff->flags |= FILE_NOSTORE;
 
     if (ff->state == FILE_STATE_OPENED && FileSize(ff) >= (uint64_t)FileMagicSize()) {
-        if (g_file_force_md5 == 0 && g_file_force_tracking == 0) {
+        if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_tracking == 0) {
             (void)FileCloseFilePtr(ff, NULL, 0,
                     (FILE_TRUNCATED|FILE_NOSTORE));
         }

src/util-file.h

@@ -189,6 +189,10 @@ void FileDisableMd5(Flow *f, uint8_t);
 void FileForceMd5Enable(void);
 int FileForceMd5(void);
 
+void FileDisableSha1(Flow *f, uint8_t);
+void FileForceSha1Enable(void);
+int FileForceSha1(void);
+
 void FileForceTrackingEnable(void);
 
 void FileStoreAllFiles(FileContainer *);

src/util-lua-common.c

@@ -593,8 +593,19 @@ static int LuaCallbackFileInfoPushToStackFromFile(lua_State *luastate, const Fil
             strlcat(md5, one, sizeof(md5));
         }
     }
+    char sha1[41] = "";
+    char *sha1ptr = sha1;
+    if (file->flags & FILE_SHA1) {
+        size_t x;
+        for (x = 0; x < sizeof(file->sha1); x++) {
+            char one[3] = "";
+            snprintf(one, sizeof(one), "%02x", file->sha1[x]);
+            strlcat(sha1, one, sizeof(sha1));
+        }
+    }
 #else
     char *md5ptr = NULL;
+    char *sha1ptr = NULL;
 #endif
 
     lua_pushnumber(luastate, file->file_id);
@@ -603,6 +614,7 @@ static int LuaCallbackFileInfoPushToStackFromFile(lua_State *luastate, const Fil
     lua_pushnumber(luastate, FileSize(file));
     lua_pushstring (luastate, file->magic);
     lua_pushstring(luastate, md5ptr);
+    lua_pushstring(luastate, sha1ptr);
     return 6;
 }