fuzz: better configure checks for MSAN building

More compatible check for rust nightly
Checks for CARGO_BUILD_TARGET
Builds release or debug mode independently

configure.ac

@@ -472,47 +472,6 @@
 # options
 
 
-    AC_ARG_ENABLE(fuzztargets,
-        AS_HELP_STRING([--enable-fuzztargets], [Enable fuzz targets]),[enable_fuzztargets=$enableval],[enable_fuzztargets=no])
-    AM_CONDITIONAL([BUILD_FUZZTARGETS], [test "x$enable_fuzztargets" = "xyes"])
-    AM_CONDITIONAL([RUST_BUILD_STD], [test "x$enable_fuzztargets" = "xyes" && echo $rust_compiler_version | grep -q nightly])
-    AC_PROG_CXX
-    AS_IF([test "x$enable_fuzztargets" = "xyes"], [
-        AC_DEFINE([FUZZ], [1], [Fuzz targets are enabled])
-        AC_DEFINE([AFLFUZZ_NO_RANDOM], [1], [Disable all use of random functions])
-        CFLAGS_ORIG=$CFLAGS
-        CFLAGS="-Werror"
-        AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[while (__AFL_LOOP(1000))]])],
-                [AC_DEFINE([AFLFUZZ_PERSISTANT_MODE], [1], [Enable AFL PERSISTANT_MODE])],
-                [])
-        CFLAGS=$CFLAGS_ORIG
-        AC_LANG_PUSH(C++)
-        tmp_saved_flags=$[]_AC_LANG_PREFIX[]FLAGS
-        AS_IF([test "x$LIB_FUZZING_ENGINE" = "x"], [
-            LIB_FUZZING_ENGINE=-fsanitize=fuzzer
-            AC_SUBST(LIB_FUZZING_ENGINE)
-        ])
-        _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $LIB_FUZZING_ENGINE"
-        AC_MSG_CHECKING([whether $CXX accepts $LIB_FUZZING_ENGINE])
-        AC_LINK_IFELSE([AC_LANG_SOURCE([[
-#include <sys/types.h>
-extern "C" int LLVMFuzzerTestOneInput(const unsigned char *Data, size_t Size);
-extern "C" int LLVMFuzzerTestOneInput(const unsigned char *Data, size_t Size) {
-(void)Data;
-(void)Size;
-return 0;
-}
-            ]])],
-            [ AC_MSG_RESULT(yes)
-              has_sanitizefuzzer=yes],
-            [ AC_MSG_RESULT(no) ]
-        )
-        _AC_LANG_PREFIX[]FLAGS=$tmp_saved_flags
-        AC_LANG_POP()
-    ])
-
-    AM_CONDITIONAL([HAS_FUZZLDFLAGS], [test "x$has_sanitizefuzzer" = "xyes"])
-
   # enable the running of unit tests
     AC_ARG_ENABLE(unittests,
            AS_HELP_STRING([--enable-unittests], [Enable compilation of the unit tests]),[enable_unittests=$enableval],[enable_unittests=no])
@@ -2662,6 +2621,51 @@ fi
     ])
     AC_SUBST(RUST_FEATURES)
 
+    AC_ARG_ENABLE(fuzztargets,
+        AS_HELP_STRING([--enable-fuzztargets], [Enable fuzz targets]),[enable_fuzztargets=$enableval],[enable_fuzztargets=no])
+    AM_CONDITIONAL([BUILD_FUZZTARGETS], [test "x$enable_fuzztargets" = "xyes"])
+    AM_CONDITIONAL([RUST_BUILD_STD], [test "x$enable_fuzztargets" = "xyes" && echo "$rust_compiler_version" | grep -q nightly])
+    AC_PROG_CXX
+    AS_IF([test "x$enable_fuzztargets" = "xyes"], [
+        AS_IF([test "x$CARGO_BUILD_TARGET" = "x" && echo "$rust_compiler_version" | grep -q nightly], [
+            CARGO_BUILD_TARGET=x86_64-unknown-linux-gnu
+            AC_SUBST(CARGO_BUILD_TARGET)
+        ])
+        AC_DEFINE([FUZZ], [1], [Fuzz targets are enabled])
+        AC_DEFINE([AFLFUZZ_NO_RANDOM], [1], [Disable all use of random functions])
+        CFLAGS_ORIG=$CFLAGS
+        CFLAGS="-Werror"
+        AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[while (__AFL_LOOP(1000))]])],
+                [AC_DEFINE([AFLFUZZ_PERSISTANT_MODE], [1], [Enable AFL PERSISTANT_MODE])],
+                [])
+        CFLAGS=$CFLAGS_ORIG
+        AC_LANG_PUSH(C++)
+        tmp_saved_flags=$[]_AC_LANG_PREFIX[]FLAGS
+        AS_IF([test "x$LIB_FUZZING_ENGINE" = "x"], [
+            LIB_FUZZING_ENGINE=-fsanitize=fuzzer
+            AC_SUBST(LIB_FUZZING_ENGINE)
+        ])
+        _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $LIB_FUZZING_ENGINE"
+        AC_MSG_CHECKING([whether $CXX accepts $LIB_FUZZING_ENGINE])
+        AC_LINK_IFELSE([AC_LANG_SOURCE([[
+#include <sys/types.h>
+extern "C" int LLVMFuzzerTestOneInput(const unsigned char *Data, size_t Size);
+extern "C" int LLVMFuzzerTestOneInput(const unsigned char *Data, size_t Size) {
+(void)Data;
+(void)Size;
+return 0;
+}
+            ]])],
+            [ AC_MSG_RESULT(yes)
+              has_sanitizefuzzer=yes],
+            [ AC_MSG_RESULT(no) ]
+        )
+        _AC_LANG_PREFIX[]FLAGS=$tmp_saved_flags
+        AC_LANG_POP()
+    ])
+
+    AM_CONDITIONAL([HAS_FUZZLDFLAGS], [test "x$has_sanitizefuzzer" = "xyes"])
+
 # get revision
     if test -f ./revision; then
         REVISION=`cat ./revision`

rust/Makefile.am

@@ -7,12 +7,12 @@ if HAVE_CARGO_VENDOR
 EXTRA_DIST +=	vendor
 endif
 
-if RUST_BUILD_STD
-RELEASE = -Z build-std
-else
 if !DEBUG
 RELEASE = --release
 endif
+
+if RUST_BUILD_STD
+NIGHTLY_ARGS = -Z build-std
 endif
 
 if HAVE_LUA
@@ -42,7 +42,7 @@ else
 	@rustup_home@ \
 		CARGO_HOME="$(CARGO_HOME)" \
 		CARGO_TARGET_DIR="$(abs_top_builddir)/rust/target" \
-		$(CARGO) build $(RELEASE) \
+		$(CARGO) build $(RELEASE) $(NIGHTLY_ARGS) \
 			--features "$(RUST_FEATURES)" $(RUST_TARGET)
 endif
 	$(MAKE) gen/rust-bindings.h