ci: Make ASAN build link with lua, build lua with address sanitizer

.ci/common/build.sh

@@ -2,7 +2,8 @@ build_deps() {
   if [[ "${BUILD_32BIT}" == ON ]]; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} ${CMAKE_FLAGS_32BIT}"
   fi
-  if [[ "${FUNCTIONALTEST}" == "functionaltest-lua" ]]; then
+  if [[ "${FUNCTIONALTEST}" == "functionaltest-lua" ]] \
+     || [[ "${CLANG_SANITIZER}" == "ASAN_UBSAN" ]]; then
     DEPS_CMAKE_FLAGS="${DEPS_CMAKE_FLAGS} -DUSE_BUNDLED_LUA=ON"
   fi
 

.travis.yml

@@ -69,7 +69,9 @@ matrix:
       env: BUILD_32BIT=ON
     - os: linux
       compiler: clang-3.8
-      env: CLANG_SANITIZER=ASAN_UBSAN
+      env: >
+        CLANG_SANITIZER=ASAN_UBSAN
+        CMAKE_FLAGS="$CMAKE_FLAGS -DPREFER_LUAJIT=false"
     - os: linux
       compiler: clang-3.8
       env: CLANG_SANITIZER=TSAN

third-party/cmake/BuildLua.cmake

@@ -51,19 +51,32 @@ else()
   endif()
 endif()
 
+set(LUA_CFLAGS "-g3")
+set(LUA_LDFLAGS "")
+
+if(CLANG_ASAN_UBSAN)
+  set(LUA_CFLAGS "${LUA_CFLAGS} -fsanitize=address")
+  set(LUA_CFLAGS "${LUA_CFLAGS} -fno-omit-frame-pointer")
+  set(LUA_CFLAGS "${LUA_CFLAGS} -fno-optimize-sibling-calls")
+
+  set(LUA_LDFLAGS "${LUA_LDFLAGS} -fsanitize=address")
+endif()
+
 set(LUA_CONFIGURE_COMMAND
   sed -e "/^CC/s@gcc@${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}@"
-      -e "/^CFLAGS/s@-O2@-g3@"
+      -e "/^CFLAGS/s@-O2@${LUA_CFLAGS}@"
+      -e "/^MYLDFLAGS/s@$@${LUA_LDFLAGS}@"
       -e "s@-lreadline@@g"
       -e "s@-lhistory@@g"
       -e "s@-lncurses@@g"
       -i ${DEPS_BUILD_DIR}/src/lua/src/Makefile &&
   sed -e "/#define LUA_USE_READLINE/d"
       -i ${DEPS_BUILD_DIR}/src/lua/src/luaconf.h)
+set(LUA_INSTALL_TOP_ARG "INSTALL_TOP=${DEPS_INSTALL_DIR}")
 set(LUA_BUILD_COMMAND
-  ${MAKE_PRG} ${LUA_TARGET})
+    ${MAKE_PRG} ${LUA_INSTALL_TOP_ARG} ${LUA_TARGET})
 set(LUA_INSTALL_COMMAND
-  ${MAKE_PRG} INSTALL_TOP=${DEPS_INSTALL_DIR} install)
+    ${MAKE_PRG} ${LUA_INSTALL_TOP_ARG} install)
 
 message(STATUS "Lua target is ${LUA_TARGET}")