hash/sha1: optimize by avoiding mem alloc

Don't allocate an output buffer for each call. These buffers
would have the exact same size every time.

src/app-layer-ssl.c

@@ -463,20 +463,15 @@ static inline int TlsDecodeHSCertificateFingerprint(SSLState *ssl_state,
     if (ssl_state->server_connp.cert0_fingerprint == NULL)
         return -1;
 
-    uint8_t *hash = ComputeSHA1((uint8_t *)input, cert_len);
-    if (hash == NULL)
-        return 0;
-
-    int i, x;
-    for (i = 0, x = 0; x < SHA1_LENGTH; x++)
-    {
-        i += snprintf(ssl_state->server_connp.cert0_fingerprint + i,
-                      SHA1_STRING_LENGTH - i, i == 0 ? "%02x" : ":%02x",
-                      *(hash + x));
+    uint8_t hash[SHA1_LENGTH];
+    if (ComputeSHA1(input, cert_len, hash, sizeof(hash)) == 1) {
+        for (int i = 0, x = 0; x < SHA1_LENGTH; x++)
+        {
+            i += snprintf(ssl_state->server_connp.cert0_fingerprint + i,
+                    SHA1_STRING_LENGTH - i, i == 0 ? "%02x" : ":%02x",
+                    hash[x]);
+        }
     }
-
-    SCFree(hash);
-
     return 0;
 }
 

src/util-crypt.c

@@ -224,40 +224,45 @@ static int Sha1Done(HashState * md, unsigned char *out)
     return SC_SHA_1_OK;
 }
 
-unsigned char* ComputeSHA1(unsigned char* buff, int bufflen)
+/** \brief calculate SHA1 hash
+ *  \retval int 1 for success, 0 for fail
+ */
+int ComputeSHA1(const uint8_t *inbuf, size_t inbuf_len,
+        uint8_t *outbuf, size_t outbuf_size)
 {
+    if (unlikely(outbuf_size != 20))
+        return 0;
+
     HashState md;
-    unsigned char* lResult = (unsigned char*) SCMalloc((sizeof(unsigned char) * 20));
-    if (lResult == NULL)
-        return NULL;
     Sha1Init(&md);
-    Sha1Process(&md, buff, bufflen);
-    Sha1Done(&md, lResult);
-    return lResult;
+    Sha1Process(&md, inbuf, inbuf_len);
+    Sha1Done(&md, outbuf);
+    return 1;
 }
 
 #else /* HAVE_NSS */
 
-unsigned char* ComputeSHA1(unsigned char* buff, int bufflen)
+/** \brief calculate SHA1 hash
+ *  \retval int 1 for success, 0 for fail
+ */
+int ComputeSHA1(const uint8_t *inbuf, size_t inbuf_len,
+        uint8_t *outbuf, size_t outbuf_size)
 {
+    if (unlikely(outbuf_size != 20))
+        return 0;
+
     HASHContext *sha1_ctx = HASH_Create(HASH_AlgSHA1);
-    unsigned char* lResult = NULL;
-    unsigned int rlen;
     if (sha1_ctx == NULL) {
-        return NULL;
+        return 0;
     }
 
-    lResult = (unsigned char*) SCMalloc((sizeof(unsigned char) * 20));
-    if (lResult == NULL) {
-        HASH_Destroy(sha1_ctx);
-        return NULL;
-    }
     HASH_Begin(sha1_ctx);
-    HASH_Update(sha1_ctx, buff, bufflen);
-    HASH_End(sha1_ctx, lResult, &rlen, (sizeof(unsigned char) * 20));
+    HASH_Update(sha1_ctx, inbuf, inbuf_len);
+    unsigned int rlen;
+    HASH_End(sha1_ctx, outbuf, &rlen, outbuf_size);
     HASH_Destroy(sha1_ctx);
 
-    return lResult;
+    return rlen == outbuf_size;
 }
 
 #endif /* HAVE_NSS */

src/util-crypt.h

@@ -80,7 +80,8 @@ typedef union HashState_ {
 
 #endif /* don't HAVE_NSS */
 
-unsigned char* ComputeSHA1(unsigned char* buff, int bufflen);
+int ComputeSHA1(const uint8_t * inbuf, size_t inbuf_len,
+        uint8_t *outbuf, size_t outbuf_len);
 int Base64Encode(const unsigned char *in,  unsigned long inlen, unsigned char *out, unsigned long *outlen);
 
 #endif /* UTIL_CRYPT_H_ */