Support for reconnecting unix domain socket log files

Issue #1423

src/util-logopenfile.c

@@ -36,11 +36,12 @@
 
 /** \brief connect to the indicated local stream socket, logging any errors
  *  \param path filesystem path to connect to
+ *  \param log_err, non-zero if connect failure should be logged.
  *  \retval FILE* on success (fdopen'd wrapper of underlying socket)
  *  \retval NULL on error
  */
 static FILE *
-SCLogOpenUnixSocketFp(const char *path, int sock_type)
+SCLogOpenUnixSocketFp(const char *path, int sock_type, int log_err)
 {
     struct sockaddr_un sun;
     int s = -1;
@@ -64,8 +65,10 @@ SCLogOpenUnixSocketFp(const char *path, int sock_type)
     return ret;
 
 err:
-    SCLogError(SC_ERR_SOCKET, "Error connecting to socket \"%s\": %s",
-               path, strerror(errno));
+    if (log_err)
+        SCLogWarning(SC_ERR_SOCKET,
+            "Error connecting to socket \"%s\": %s (will keep trying)",
+            path, strerror(errno));
 
     if (s >= 0)
         close(s);
@@ -73,6 +76,52 @@ err:
     return NULL;
 }
 
+/**
+ * \brief Attempt to reconnect a disconnected (or never-connected) Unix domain socket.
+ * \retval 1 if it is now connected; otherwise 0
+ */
+static int SCLogUnixSocketReconnect(LogFileCtx *log_ctx)
+{
+    int disconnected = 0;
+    if (log_ctx->fp) {
+        SCLogWarning(SC_ERR_SOCKET,
+            "Write error on Unix socket \"%s\": %s; reconnecting...",
+            log_ctx->filename, strerror(errno));
+        fclose(log_ctx->fp);
+        log_ctx->fp = NULL;
+        log_ctx->reconn_timer = 0;
+        disconnected = 1;
+    }
+
+    struct timeval tv;
+    uint64_t now;
+    gettimeofday(&tv, NULL);
+    now = (uint64_t)tv.tv_sec * 1000;
+    now += tv.tv_usec / 1000;           /* msec resolution */
+    if (log_ctx->reconn_timer != 0 &&
+            (now - log_ctx->reconn_timer) < LOGFILE_RECONN_MIN_TIME) {
+        /* Don't bother to try reconnecting too often. */
+        return 0;
+    }
+    log_ctx->reconn_timer = now;
+
+    log_ctx->fp = SCLogOpenUnixSocketFp(log_ctx->filename, log_ctx->sock_type, 0);
+    if (log_ctx->fp) {
+        /* Connected at last (or reconnected) */
+        SCLogNotice("Reconnected socket \"%s\"", log_ctx->filename);
+    } else if (disconnected) {
+        SCLogWarning(SC_ERR_SOCKET, "Reconnect failed: %s (will keep trying)",
+            strerror(errno));
+    }
+
+    return log_ctx->fp ? 1 : 0;
+}
+
+/**
+ * \brief Write buffer to log file.
+ * \retval 0 on failure; otherwise, the return value of fwrite (number of
+ * characters successfully written).
+ */
 static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx)
 {
     /* Check for rotation. */
@@ -83,9 +132,21 @@ static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ct
 
     int ret = 0;
 
+    if (log_ctx->fp == NULL && log_ctx->is_sock)
+        SCLogUnixSocketReconnect(log_ctx);
+
     if (log_ctx->fp) {
+        clearerr(log_ctx->fp);
         ret = fwrite(buffer, buffer_len, 1, log_ctx->fp);
         fflush(log_ctx->fp);
+
+        if (ferror(log_ctx->fp) && log_ctx->is_sock) {
+            /* Error on Unix socket, maybe needs reconnect */
+            if (SCLogUnixSocketReconnect(log_ctx)) {
+                ret = fwrite(buffer, buffer_len, 1, log_ctx->fp);
+                fflush(log_ctx->fp);
+            }
+        }
     }
 
     return ret;
@@ -194,25 +255,21 @@ SCConfLogOpenGeneric(ConfNode *conf,
 
     // Now, what have we been asked to open?
     if (strcasecmp(filetype, "unix_stream") == 0) {
-        log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM);
-        if (log_ctx->fp == NULL)
-            return -1; // Error already logged by Open...Fp routine
+        /* Don't bail. May be able to connect later. */
+        log_ctx->is_sock = 1;
+        log_ctx->sock_type = SOCK_STREAM;
+        log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM, 1);
     } else if (strcasecmp(filetype, "unix_dgram") == 0) {
-        log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM);
-        if (log_ctx->fp == NULL)
-            return -1; // Error already logged by Open...Fp routine
+        /* Don't bail. May be able to connect later. */
+        log_ctx->is_sock = 1;
+        log_ctx->sock_type = SOCK_DGRAM;
+        log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM, 1);
     } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
                strcasecmp(filetype, "file") == 0) {
         log_ctx->fp = SCLogOpenFileFp(log_path, append);
         if (log_ctx->fp == NULL)
             return -1; // Error already logged by Open...Fp routine
         log_ctx->is_regular = 1;
-        log_ctx->filename = SCStrdup(log_path);
-        if (unlikely(log_ctx->filename == NULL)) {
-            SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate memory for "
-                "filename");
-            return -1;
-        }
         if (rotate) {
             OutputRegisterFileRotationFlag(&log_ctx->rotation_flag);
         }
@@ -227,6 +284,12 @@ SCConfLogOpenGeneric(ConfNode *conf,
                    "or \"unix_dgram\"",
                    conf->name);
     }
+    log_ctx->filename = SCStrdup(log_path);
+    if (unlikely(log_ctx->filename == NULL)) {
+        SCLogError(SC_ERR_MEM_ALLOC,
+            "Failed to allocate memory for filename");
+        return -1;
+    }
 
     SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype,
               filename);

src/util-logopenfile.h

@@ -56,6 +56,11 @@ typedef struct LogFileCtx_ {
     /** The name of the file */
     char *filename;
 
+    /** Handle auto-connecting / reconnecting sockets */
+    int is_sock;
+    int sock_type;
+    uint64_t reconn_timer;
+
     /**< Used by some alert loggers like the unified ones that append
      * the date onto the end of files. */
     char *prefix;
@@ -78,6 +83,9 @@ typedef struct LogFileCtx_ {
     int rotation_flag;
 } LogFileCtx;
 
+/* Min time (msecs) before trying to reconnect a Unix domain socket */
+#define LOGFILE_RECONN_MIN_TIME     500
+
 /* flags for LogFileCtx */
 #define LOGFILE_HEADER_WRITTEN 0x01
 #define LOGFILE_ALERTS_PRINTED 0x02