Add per-cpu prio handling

This patch updates affinity setting to add a support for per cpu
priority setting. In exclusive mode a thread is dedicated to a CPU.
This patch adds the ability to set the thread prio for all threads
of a family running on a given CPU.

With this patch we can write
    - detect_cpu_set:
        cpu: [ "all" ]
        mode: "exclusive" # run detect threads in these cpus
        low_prio: [ 0 ]
        medium_prio: [ "1-2" ]
        high_prio: [ 3 ]
With this configuration, detect threads assigned to cpu 0 will
have a low priority. Detect threads on cpus 1 and 2 will have
prio medium...

The previous configuration is equivalent to:
    - detect_cpu_set:
        cpu: [ "all" ]
        mode: "exclusive" # run detect threads in these cpus
        low_prio: [ 0 ]
        high_prio: [ 3 ]
        prio: "medium"
because the prio value is used a default.

Signed-off-by: Eric Leblond <eric@regit.org>

src/tm-threads.c

@@ -799,10 +799,22 @@ TmEcode TmThreadSetupOptions(ThreadVars *tv) {
         if (taf->mode_flag == EXCLUSIVE_AFFINITY) {
             int cpu = AffinityGetNextCPU(taf);
             SetCPUAffinity(cpu);
+            /* If CPU is in a set overwrite the default thread prio */
+            if (CPU_ISSET(cpu, &taf->lowprio_cpu)) {
+                tv->thread_priority = PRIO_LOW;
+            } else if (CPU_ISSET(cpu, &taf->medprio_cpu)) {
+                tv->thread_priority = PRIO_MEDIUM;
+            }  else if (CPU_ISSET(cpu, &taf->hiprio_cpu)) {
+                tv->thread_priority = PRIO_HIGH;
+            } else {
+                tv->thread_priority = taf->prio;
+            }
+            SCLogInfo("Setting prio %d for \"%s\" Module to cpu/core %"PRIu16", thread id %lu",
+                  tv->thread_priority, tv->name, cpu, SCGetThreadIdLong());
         } else {
             SetCPUAffinitySet(&taf->cpu_set);
+            tv->thread_priority = taf->prio;
         }
-        tv->thread_priority = taf->prio;
         TmThreadSetPrio(tv);
     }
     return TM_ECODE_OK;

src/util-affinity.c

@@ -115,6 +115,59 @@ static void AffinitySetupInit()
     return;
 }
 
+static void build_cpuset(ConfNode *node, cpu_set_t *cpu)
+{
+    ConfNode *lnode;
+    TAILQ_FOREACH(lnode, &node->head, next) {
+        int i;
+        long int a,b;
+        int stop = 0;
+        if (!strcmp(lnode->val, "all")) {
+            a = 0;
+            b = UtilCpuGetNumProcessorsConfigured();
+            stop = 1;
+        } else if (index(lnode->val, '-') != NULL) {
+            char *sep = index(lnode->val, '-');
+            char *end;
+            a = strtoul(lnode->val, &end, 10);
+            if (end != sep) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "invalid cpu range (start invalid): \"%s\"",
+                        lnode->val);
+                exit(EXIT_FAILURE);
+            }
+            b = strtol(sep + 1, &end, 10);
+            if (end != sep + strlen(sep)) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "invalid cpu range (end invalid): \"%s\"",
+                        lnode->val);
+                exit(EXIT_FAILURE);
+            }
+            if (a > b) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "invalid cpu range (bad order): \"%s\"",
+                        lnode->val);
+                exit(EXIT_FAILURE);
+            }
+        } else {
+            char *end;
+            a = strtoul(lnode->val, &end, 10);
+            if (end != lnode->val + strlen(lnode->val)) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "invalid cpu range (not an integer): \"%s\"",
+                        lnode->val);
+                exit(EXIT_FAILURE);
+            }
+            b = a;
+        }
+        for (i = a; i<= b; i++) {
+            CPU_SET(i, cpu);
+        }
+        if (stop)
+            break;
+    }
+}
+
 /**
  * \brief Extract cpu affinity configuration from current config file
  */
@@ -135,7 +188,6 @@ void AffinitySetupLoadFromConfig()
     TAILQ_FOREACH(affinity, &root->head, next) {
         ThreadsAffinityType *taf = GetAffinityTypeFromName(affinity->val);
         ConfNode *node = NULL;
-        ConfNode *lnode;
 
         if (taf == NULL) {
             SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown cpu_affinity type");
@@ -144,63 +196,40 @@ void AffinitySetupLoadFromConfig()
             SCLogInfo("Found affinity definition for \"%s\"",
                       affinity->val);
         }
-        CPU_ZERO(&taf->cpu_set);
 
+        CPU_ZERO(&taf->cpu_set);
         node = ConfNodeLookupChild(affinity->head.tqh_first, "cpu");
-
         if (node == NULL) {
             SCLogInfo("unable to find 'cpu'");
         } else {
-            TAILQ_FOREACH(lnode, &node->head, next) {
-                int i;
-                long int a,b;
-                int stop = 0;
-                if (!strcmp(lnode->val, "all")) {
-                    a = 0;
-                    b = UtilCpuGetNumProcessorsConfigured();
-                    stop = 1;
-                } else if (index(lnode->val, '-') != NULL) {
-                    char *sep = index(lnode->val, '-');
-                    char *end;
-                    a = strtoul(lnode->val, &end, 10);
-                    if (end != sep) {
-                        SCLogError(SC_ERR_INVALID_ARGUMENT,
-                                "invalid cpu range (start invalid): \"%s\"",
-                                lnode->val);
-                        exit(EXIT_FAILURE);
-                    }
-                    b = strtol(sep + 1, &end, 10);
-                    if (end != sep + strlen(sep)) {
-                        SCLogError(SC_ERR_INVALID_ARGUMENT,
-                                "invalid cpu range (end invalid): \"%s\"",
-                                lnode->val);
-                        exit(EXIT_FAILURE);
-                    }
-                    if (a > b) {
-                        SCLogError(SC_ERR_INVALID_ARGUMENT,
-                                "invalid cpu range (bad order): \"%s\"",
-                                lnode->val);
-                        exit(EXIT_FAILURE);
-                    }
-                } else {
-                    char *end;
-                    a = strtoul(lnode->val, &end, 10);
-                    if (end != lnode->val + strlen(lnode->val)) {
-                        SCLogError(SC_ERR_INVALID_ARGUMENT,
-                                "invalid cpu range (not an integer): \"%s\"",
-                                lnode->val);
-                        exit(EXIT_FAILURE);
-                    }
-                    b = a;
-                }
-                for (i = a; i<= b; i++) {
-                    CPU_SET(i, &taf->cpu_set);
-                }
-                if (stop)
-                    break;
-            }
+            build_cpuset(node, &taf->cpu_set);
+        }
+
+        CPU_ZERO(&taf->lowprio_cpu);
+        node = ConfNodeLookupChild(affinity->head.tqh_first, "low_prio");
+        if (node == NULL) {
+            SCLogDebug("unable to find 'low_prio' using default value");
+        } else {
+            build_cpuset(node, &taf->lowprio_cpu);
+        }
+
+        CPU_ZERO(&taf->medprio_cpu);
+        node = ConfNodeLookupChild(affinity->head.tqh_first, "medium_prio");
+        if (node == NULL) {
+            SCLogDebug("unable to find 'medium_prio' using default value");
+        } else {
+            build_cpuset(node, &taf->medprio_cpu);
+        }
+
+        CPU_ZERO(&taf->hiprio_cpu);
+        node = ConfNodeLookupChild(affinity->head.tqh_first, "high_prio");
+        if (node == NULL) {
+            SCLogDebug("unable to find 'high_prio' using default value");
+        } else {
+            build_cpuset(node, &taf->hiprio_cpu);
         }
 
+
         node = ConfNodeLookupChild(affinity->head.tqh_first, "mode");
         if (node != NULL) {
             if (!strcmp(node->val, "exclusive")) {

src/util-affinity.h

@@ -48,6 +48,9 @@ typedef struct ThreadsAffinityType_ {
     cpu_set_t cpu_set;
     uint8_t mode_flag;
     uint8_t prio;
+    cpu_set_t lowprio_cpu;
+    cpu_set_t medprio_cpu;
+    cpu_set_t hiprio_cpu;
     uint16_t lcpu; /* use by exclusive mode */
 } ThreadsAffinityType;
 

suricata.yaml

@@ -174,12 +174,19 @@ threading:
     - detect_cpu_set:
         cpu: [ "all" ]
         mode: "exclusive" # run detect threads in these cpus
+        low_prio: [ 0 ]
+        medium_prio: [ "1-2" ]
+        high_prio: [ 3 ]
+        prio: "medium"
     - verdict_cpu_set:
         cpu: [ 0 ]
+        prio: "high"
     - reject_cpu_set:
         cpu: [ 0 ]
+        prio: "low"
     - output_cpu_set:
         cpu: [ "all" ]
+        prio: "medium"
   #
   # By default Suricata creates one "detect" thread per available CPU/CPU core.
   # This setting allows controlling this behaviour. A ratio setting of 2 will