Fix not decreasing the flow use_cnt reference counter in some cases from the app layer detection code. This caused some streams to never fully time out and thus clutter up the flow table and session pool.

src/detect.c

@@ -489,6 +489,8 @@ inline SigGroupHead *SigMatchSignaturesGetSgh(ThreadVars *th_v, DetectEngineCtx
 /** \brief application layer detection
  *
  *  \param sgh signature group head for this proto/addrs/ports
+ *  \warning Make sure to exit this function using "goto end" if the flow
+ *           use_cnt has already been incremented.
  */
 static int SigMatchSignaturesAppLayer(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, SigGroupHead *sgh, Packet *p)
 {
@@ -522,11 +524,11 @@ static int SigMatchSignaturesAppLayer(ThreadVars *th_v, DetectEngineCtx *de_ctx,
 
     if (alproto == ALPROTO_UNKNOWN) {
         SCLogDebug("application layer state proto still unknown");
-        SCReturnInt(0);
+        goto end;
     }
     if (alstate == NULL) {
         SCLogDebug("no application layer state to detect");
-        SCReturnInt(0);
+        goto end;
     }
 
     if (p->flowflags & FLOW_PKT_TOSERVER) {
@@ -653,6 +655,7 @@ static int SigMatchSignaturesAppLayer(ThreadVars *th_v, DetectEngineCtx *de_ctx,
         }
     }
 
+end:
     SCMutexLock(&p->flow->m);
     p->flow->use_cnt--;
     SCMutexUnlock(&p->flow->m);

src/flow-util.c

@@ -56,7 +56,7 @@ void FlowFree(Flow *f)
  *  \param   proto  protocol which is needed to be mapped
  */
 
-int FlowGetProtoMapping(uint8_t proto) {
+uint8_t FlowGetProtoMapping(uint8_t proto) {
 
     switch (proto) {
         case IPPROTO_TCP:

src/flow-util.h

@@ -24,7 +24,7 @@
 
 Flow *FlowAlloc(void);
 void FlowFree(Flow *);
-int FlowGetProtoMapping(uint8_t);
+uint8_t FlowGetProtoMapping(uint8_t);
 void FlowInit(Flow *, Packet *);
 
 #endif /* __FLOW_UTIL_H__ */

src/flow.c

@@ -67,9 +67,9 @@ void FlowUpdateQueue(Flow *f)
             uint8_t state = flow_proto[f->protomap].GetProtoState(f->protoctx);
             if (state == FLOW_STATE_CLOSED) {
                 f->flags |= FLOW_CLOSED_LIST; /* transition */
-                f->flags &= ~FLOW_EST_LIST;
+                f->flags &=~ FLOW_EST_LIST;
 
-                //printf("FlowUpdateQueue %p was put into closing queue ts %"PRIuMAX"\n", f, (uintmax_t)f->lastts.tv_sec);
+                SCLogDebug("flow %p was put into closing queue ts %"PRIuMAX"", f, (uintmax_t)f->lastts.tv_sec);
                 FlowRequeue(f, &flow_est_q[f->protomap], &flow_close_q[f->protomap]);
             } else {
                 /* Pull and put back -- this way the flows on
@@ -659,7 +659,7 @@ void *FlowManagerThread(void *td)
         sleeping += 10;
     }
 
-    SCLogInfo("%" PRIu32 " new flows, %" PRIu32 " established flows were timed out", new_cnt, established_cnt);
+    SCLogInfo("%" PRIu32 " new flows, %" PRIu32 " established flows were timed out, %"PRIu32"", new_cnt, established_cnt, closing_cnt);
     pthread_exit((void *) 0);
 }
 

src/stream-tcp.c

@@ -173,12 +173,12 @@ static void StreamTcpSessionPktFree (Packet *p)
     SCMutexUnlock(&ssn_pool_mutex);
 
     p->flow->protoctx = NULL;
-*/
 #ifdef DEBUG
     SCMutexLock(&ssn_pool_cnt_mutex);
     ssn_pool_cnt--;
     SCMutexUnlock(&ssn_pool_cnt_mutex);
 #endif
+*/
 
     SCReturn;
 }
@@ -2561,25 +2561,30 @@ static int ValidReset(TcpSession *ssn, Packet *p)
 /**
  *  \brief  Function to return the FLOW state depending upon the TCP session state.
  *
- *  \param   s    TCP session of which the state has to be returned
- *  \retval       The FLOW_STATE_ depends upon the TCP sesison state, default is
- *                FLOW_STATE_CLOSED
+ *  \param   s      TCP session of which the state has to be returned
+ *  \retval  state  The FLOW_STATE_ depends upon the TCP sesison state, default is
+ *                  FLOW_STATE_CLOSED
  */
 
 int StreamTcpGetFlowState(void *s)
 {
+    SCEnter();
+
     TcpSession *ssn = (TcpSession *)s;
-    if (ssn == NULL)
-        return FLOW_STATE_CLOSED;
+    if (ssn == NULL) {
+        SCReturnInt(FLOW_STATE_CLOSED);
+    }
 
     switch(ssn->state) {
         case TCP_NONE:
         case TCP_SYN_SENT:
         case TCP_SYN_RECV:
         case TCP_LISTEN:
-            return FLOW_STATE_NEW;
+            SCReturnInt(FLOW_STATE_NEW);
+
         case TCP_ESTABLISHED:
-            return FLOW_STATE_ESTABLISHED;
+            SCReturnInt(FLOW_STATE_ESTABLISHED);
+
         case TCP_FIN_WAIT1:
         case TCP_FIN_WAIT2:
         case TCP_CLOSING:
@@ -2587,9 +2592,10 @@ int StreamTcpGetFlowState(void *s)
         case TCP_TIME_WAIT:
         case TCP_CLOSE_WAIT:
         case TCP_CLOSED:
-            return FLOW_STATE_CLOSED;
+            SCReturnInt(FLOW_STATE_CLOSED);
     }
-    return FLOW_STATE_CLOSED;
+
+    SCReturnInt(FLOW_STATE_CLOSED);
 }
 
 /**