detect: fix overflows in SetupU8Hash

For instance ">255" resulted in overflow

(cherry picked from commit 2d765d6c68)

src/detect-engine-prefilter-common.c

@@ -290,29 +290,34 @@ static void SetupU8Hash(DetectEngineCtx *de_ctx, HashListTable *hash_table,
                 break;
             case PREFILTER_U8HASH_MODE_LT:
             {
-                uint8_t v = ctx->v1.u8[1] - 1;
-                do {
+                uint8_t v = ctx->v1.u8[1];
+                while (v > 0) {
+                    v--;
                     counts[v] += ctx->cnt;
-                } while (v--);
+                }
 
                 break;
             }
             case PREFILTER_U8HASH_MODE_GT:
             {
-                int v = ctx->v1.u8[1] + 1;
-                do {
+                uint8_t v = ctx->v1.u8[1];
+                while (v < UINT8_MAX) {
+                    v++;
                     counts[v] += ctx->cnt;
-                } while (++v < 256);
+                }
 
                 break;
             }
             case PREFILTER_U8HASH_MODE_RA:
             {
-                int v = ctx->v1.u8[1] + 1;
-                do {
-                    counts[v] += ctx->cnt;
-                } while (++v < ctx->v1.u8[2]);
-
+                if (ctx->v1.u8[1] < ctx->v1.u8[2]) {
+                    // ctx->v1.u8[1] is not UINT8_MAX
+                    uint8_t v = ctx->v1.u8[1] + 1;
+                    while (v < ctx->v1.u8[2]) {
+                        counts[v] += ctx->cnt;
+                        v++;
+                    }
+                }
                 break;
             }
         }