From 66394367f773f565da392f40de82f3d773dd28b9 Mon Sep 17 00:00:00 2001 From: John Szakmeister Date: Sat, 15 Mar 2014 03:22:52 -0400 Subject: [PATCH] Workaround the broken progress feedback in some versions of CMake. Underneath the hood, CMake uses libcurl and libcurl has had a number of issues regarding progress feedback. In one sample run against Travis CI, we ended up with nearly 3,000 lines of progress output for a single download. Unfortunately, CMake doesn't have the download and extract steps separate, so we have some extra work that we have to do. Much of the content was taken from the ExternalProject.cmake and it's template for generating the content of the download and extract CMake files. --- third-party/CMakeLists.txt | 42 +++++- .../cmake/DownloadAndExtractFile.cmake | 123 ++++++++++++++++++ 2 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 third-party/cmake/DownloadAndExtractFile.cmake diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 2b41666c65..854a223e96 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -42,12 +42,28 @@ endif() include(ExternalProject) +set(LIBUV_URL https://github.com/joyent/libuv/archive/v0.11.21.tar.gz) +set(LIBUV_MD5 bc334c8da8618754ce9e2a4c5be73487) + +set(LUAJIT_URL http://luajit.org/download/LuaJIT-2.0.2.tar.gz) +set(LUAJIT_MD5 112dfb82548b03377fbefbba2e0e3a5b) + +set(LUAROCKS_URL http://github.com/keplerproject/luarocks/archive/v2.1.2.tar.gz) +set(LUAROCKS_MD5 df591c00a85d51fb754ec08c77896aad) + if(USE_BUNDLED_LIBUV) ExternalProject_Add(libuv PREFIX ${DEPS_BUILD_DIR} - URL https://github.com/joyent/libuv/archive/v0.11.21.tar.gz - URL_MD5 bc334c8da8618754ce9e2a4c5be73487 + URL ${LIBUV_URL} + URL_MD5 ${LIBUV_MD5} DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/libuv + DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/libuv + -DURL=${LIBUV_URL} + -DEXPECTED_MD5=${LIBUV_MD5} + -DTARGET=libuv + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake CONFIGURE_COMMAND sh ${DEPS_BUILD_DIR}/src/libuv/autogen.sh && ${DEPS_BUILD_DIR}/src/libuv/configure --with-pic --disable-shared --prefix=${DEPS_INSTALL_DIR} CC=${CMAKE_C_COMPILER} @@ -58,9 +74,16 @@ endif() if(USE_BUNDLED_LUAJIT) ExternalProject_Add(luajit PREFIX ${DEPS_BUILD_DIR} - URL http://luajit.org/download/LuaJIT-2.0.2.tar.gz - URL_MD5 112dfb82548b03377fbefbba2e0e3a5b + URL ${LUAJIT_URL} + URL_MD5 ${LUAJIT_MD5} DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/luajit + DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/luajit + -DURL=${LUAJIT_URL} + -DEXPECTED_MD5=${LUAJIT_MD5} + -DTARGET=luajit + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake CONFIGURE_COMMAND "" BUILD_IN_SOURCE 1 BUILD_COMMAND "" @@ -79,9 +102,16 @@ if(USE_BUNDLED_LUAROCKS) endif() ExternalProject_Add(luarocks PREFIX ${DEPS_BUILD_DIR} - URL http://github.com/keplerproject/luarocks/archive/v2.1.2.tar.gz - URL_MD5 df591c00a85d51fb754ec08c77896aad + URL ${LUAROCKS_URL} + URL_MD5 ${LUAROCKS_MD5} DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/luarocks + DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/luarocks + -DURL=${LUAROCKS_URL} + -DEXPECTED_MD5=${LUAROCKS_MD5} + -DTARGET=luarocks + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake BUILD_IN_SOURCE 1 CONFIGURE_COMMAND ${DEPS_BUILD_DIR}/src/luarocks/configure --prefix=${DEPS_INSTALL_DIR} --force-config ${LUAROCKS_OPTS} diff --git a/third-party/cmake/DownloadAndExtractFile.cmake b/third-party/cmake/DownloadAndExtractFile.cmake new file mode 100644 index 0000000000..e7a01b8a87 --- /dev/null +++ b/third-party/cmake/DownloadAndExtractFile.cmake @@ -0,0 +1,123 @@ +if(NOT DEFINED PREFIX) + message(FATAL_ERROR "PREFIX must be defined.") +endif() + +if(NOT DEFINED URL) + message(FATAL_ERROR "URL must be defined.") +endif() + +if(NOT DEFINED DOWNLOAD_DIR) + message(FATAL_ERROR "DOWNLOAD_DIR must be defined.") +endif() + +if(NOT DEFINED EXPECTED_MD5) + message(FATAL_ERROR "EXPECTED_MD5 must be defined.") +endif() + +if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET must be defined.") +endif() + +# Taken from ExternalProject_Add. Let's hope we can drop this one day when +# ExternalProject_Add allows you to disable SHOW_PROGRESS on the file download. +if(TIMEOUT) + set(timeout_args TIMEOUT ${timeout}) + set(timeout_msg "${timeout} seconds") +else() + set(timeout_args "# no TIMEOUT") + set(timeout_msg "none") +endif() + +string(REGEX MATCH "[^/\\?]*$" fname "${URL}") +if(NOT "${fname}" MATCHES "(\\.|=)(bz2|tar|tgz|tar\\.gz|zip)$") + string(REGEX MATCH "([^/\\?]+(\\.|=)(bz2|tar|tgz|tar\\.gz|zip))/.*$" match_result "${URL}") + set(fname "${CMAKE_MATCH_1}") +endif() +if(NOT "${fname}" MATCHES "(\\.|=)(bz2|tar|tgz|tar\\.gz|zip)$") + message(FATAL_ERROR "Could not extract tarball filename from url:\n ${url}") +endif() +string(REPLACE ";" "-" fname "${fname}") + +set(file ${DOWNLOAD_DIR}/${fname}) +message(STATUS "file: ${file}") + +message(STATUS "downloading... + src='${URL}' + dst='${file}' + timeout='${timeout_msg}'") + +file(DOWNLOAD ${URL} ${file} + ${timeout_args} + EXPECTED_MD5 ${EXPECTED_MD5} + STATUS status + LOG log) + +list(GET status 0 status_code) +list(GET status 1 status_string) + +if(NOT status_code EQUAL 0) + message(FATAL_ERROR "error: downloading '${URL}' failed + status_code: ${status_code} + status_string: ${status_string} + log: ${log} +") +endif() + +message(STATUS "downloading... done") + +set(SRC_DIR ${PREFIX}/src/${TARGET}) + +# Slurped from a generated extract-TARGET.cmake file. +message(STATUS "extracting... + src='${file}' + dst='${SRC_DIR}'") + +if(NOT EXISTS "${file}") + message(FATAL_ERROR "error: file to extract does not exist: '${file}'") +endif() + +# Prepare a space for extracting: +# +set(i 1234) +while(EXISTS "${SRC_DIR}/../ex-${TARGET}${i}") + math(EXPR i "${i} + 1") +endwhile() +set(ut_dir "${SRC_DIR}/../ex-${TARGET}${i}") +file(MAKE_DIRECTORY "${ut_dir}") + +# Extract it: +# +message(STATUS "extracting... [tar xfz]") +execute_process(COMMAND ${CMAKE_COMMAND} -E tar xfz ${file} + WORKING_DIRECTORY ${ut_dir} + RESULT_VARIABLE rv) + +if(NOT rv EQUAL 0) + message(STATUS "extracting... [error clean up]") + file(REMOVE_RECURSE "${ut_dir}") + message(FATAL_ERROR "error: extract of '${file}' failed") +endif() + +# Analyze what came out of the tar file: +# +message(STATUS "extracting... [analysis]") +file(GLOB contents "${ut_dir}/*") +list(LENGTH contents n) +if(NOT n EQUAL 1 OR NOT IS_DIRECTORY "${contents}") + set(contents "${ut_dir}") +endif() + +# Move "the one" directory to the final directory: +# +message(STATUS "extracting... [rename]") +file(REMOVE_RECURSE ${SRC_DIR}) +get_filename_component(contents ${contents} ABSOLUTE) +file(RENAME ${contents} ${SRC_DIR}) + +# Clean up: +# +message(STATUS "extracting... [clean up]") +file(REMOVE_RECURSE "${ut_dir}") + +message(STATUS "extracting... done") +