cmake/OpenCVDownload.cmake

   1 #
   2 #  Download and optionally unpack a file
   3 #
   4 #  ocv_download(FILENAME p HASH h URL u1 [u2 ...] DESTINATION_DIR d [ID id] [STATUS s] [UNPACK] [RELATIVE_URL])
   5 #    FILENAME - filename
   6 #    HASH - MD5 hash
   7 #    URL - full download url (first nonempty value will be chosen)
   8 #    DESTINATION_DIR - file will be copied to this directory
   9 #    ID     - identifier for project/group of downloaded files
  10 #    STATUS - passed variable will be updated in parent scope,
  11 #             function will not fail the build in case of download problem if this option is provided,
  12 #             but will fail in case when other operations (copy, remove, etc.) failed
  13 #    UNPACK - downloaded file will be unpacked to DESTINATION_DIR
  14 #    RELATIVE_URL - if set, then URL is treated as a base, and FILENAME will be appended to it
  15 #  Note: uses OPENCV_DOWNLOAD_PATH folder as cache, default is <opencv>/.cache
  16
  17 set(HELP_OPENCV_DOWNLOAD_PATH "Cache directory for downloaded files")
  18 if(DEFINED ENV{OPENCV_DOWNLOAD_PATH})
  19   set(OPENCV_DOWNLOAD_PATH "$ENV{OPENCV_DOWNLOAD_PATH}" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}")
  20 endif()
  21 set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}")
  22 set(OPENCV_DOWNLOAD_LOG "${OpenCV_BINARY_DIR}/CMakeDownloadLog.txt")
  23
  24 # Init download cache directory and log file
  25 if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}")
  26   file(MAKE_DIRECTORY ${OPENCV_DOWNLOAD_PATH})
  27 endif()
  28 if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}/.gitignore")
  29   file(WRITE "${OPENCV_DOWNLOAD_PATH}/.gitignore" "*\n")
  30 endif()
  31 file(WRITE "${OPENCV_DOWNLOAD_LOG}" "use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n")
  32
  33
  34 function(ocv_download)
  35   cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN})
  36
  37   function(ocv_download_log)
  38     file(APPEND "${OPENCV_DOWNLOAD_LOG}" "${ARGN}\n")
  39   endfunction()
  40
  41   ocv_assert(DL_FILENAME)
  42   ocv_assert(DL_HASH)
  43   ocv_assert(DL_URL)
  44   ocv_assert(DL_DESTINATION_DIR)
  45   if((NOT " ${DL_UNPARSED_ARGUMENTS}" STREQUAL " ")
  46     OR DL_FILENAME STREQUAL ""
  47     OR DL_HASH STREQUAL ""
  48     OR DL_URL STREQUAL ""
  49     OR DL_DESTINATION_DIR STREQUAL ""
  50   )
  51     set(msg_level FATAL_ERROR)
  52     if(DEFINED DL_STATUS)
  53       set(${DL_STATUS} FALSE PARENT_SCOPE)
  54       set(msg_level WARNING)
  55     endif()
  56     message(${msg_level} "ERROR: ocv_download() unsupported arguments: ${ARGV}")
  57     return()
  58   endif()
  59
  60   if(DEFINED DL_STATUS)
  61     set(${DL_STATUS} TRUE PARENT_SCOPE)
  62   endif()
  63
  64   # Check CMake cache for already processed tasks
  65   string(FIND "${DL_DESTINATION_DIR}" "${CMAKE_BINARY_DIR}" DL_BINARY_PATH_POS)
  66   if(DL_BINARY_PATH_POS EQUAL 0)
  67     set(__file_id "${DL_DESTINATION_DIR}/${DL_FILENAME}")
  68     file(RELATIVE_PATH __file_id "${CMAKE_BINARY_DIR}" "${__file_id}")
  69     string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" __file_id "${__file_id}")
  70     if(DL_ID)
  71       string(TOUPPER ${DL_ID} __id)
  72       string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" __id "${__id}")
  73       set(OCV_DOWNLOAD_HASH_NAME "OCV_DOWNLOAD_${__id}_HASH_${__file_id}")
  74     else()
  75       set(OCV_DOWNLOAD_HASH_NAME "OCV_DOWNLOAD_HASH_${__file_id}")
  76     endif()
  77     if(" ${${OCV_DOWNLOAD_HASH_NAME}}" STREQUAL " ${DL_HASH}")
  78       ocv_download_log("#match_hash_in_cmake_cache \"${OCV_DOWNLOAD_HASH_NAME}\"")
  79       return()
  80     endif()
  81     unset("${OCV_DOWNLOAD_HASH_NAME}" CACHE)
  82   else()
  83     set(OCV_DOWNLOAD_HASH_NAME "")
  84     #message(WARNING "Download destination is not in CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}: ${DL_DESTINATION_DIR}")
  85   endif()
  86
  87   # Select first non-empty url
  88   foreach(url ${DL_URL})
  89     if(url)
  90       set(DL_URL "${url}")
  91       break()
  92     endif()
  93   endforeach()
  94
  95   # Append filename to url if needed
  96   if(DL_RELATIVE_URL)
  97     set(DL_URL "${DL_URL}${DL_FILENAME}")
  98   endif()
  99
 100   set(mode "copy")
 101   if(DL_UNPACK)
 102     set(mode "unpack")
 103   endif()
 104
 105   # Log all calls to file
 106   ocv_download_log("do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"")
 107   # ... and to console
 108   set(__msg_prefix "")
 109   if(DL_ID)
 110     set(__msg_prefix "${DL_ID}: ")
 111   endif()
 112   message(STATUS "${__msg_prefix}Download: ${DL_FILENAME}")
 113
 114   # Copy mode: check if copy destination exists and is correct
 115   if(NOT DL_UNPACK)
 116     set(COPY_DESTINATION "${DL_DESTINATION_DIR}/${DL_FILENAME}")
 117     if(EXISTS "${COPY_DESTINATION}")
 118       ocv_download_log("#check_md5 \"${COPY_DESTINATION}\"")
 119       file(MD5 "${COPY_DESTINATION}" target_md5)
 120       if(target_md5 STREQUAL DL_HASH)
 121         ocv_download_log("#match_md5 \"${COPY_DESTINATION}\" \"${target_md5}\"")
 122         if(OCV_DOWNLOAD_HASH_NAME)
 123           set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "")
 124         endif()
 125         return()
 126       endif()
 127       ocv_download_log("#mismatch_md5 \"${COPY_DESTINATION}\" \"${target_md5}\"")
 128     else()
 129       ocv_download_log("#missing \"${COPY_DESTINATION}\"")
 130     endif()
 131   endif()
 132
 133   # Check cache first
 134   if(DL_ID)
 135     string(TOLOWER "${DL_ID}" __id)
 136     string(REGEX REPLACE "[^a-zA-Z0-9_/ ]" "_" __id "${__id}")
 137     set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${__id}/${DL_HASH}-${DL_FILENAME}")
 138   else()
 139     set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${DL_HASH}-${DL_FILENAME}")
 140   endif()
 141   if(EXISTS "${CACHE_CANDIDATE}")
 142     ocv_download_log("#check_md5 \"${CACHE_CANDIDATE}\"")
 143     file(MD5 "${CACHE_CANDIDATE}" target_md5)
 144     if(NOT target_md5 STREQUAL DL_HASH)
 145       ocv_download_log("#mismatch_md5 \"${CACHE_CANDIDATE}\" \"${target_md5}\"")
 146       ocv_download_log("#delete \"${CACHE_CANDIDATE}\"")
 147       file(REMOVE ${CACHE_CANDIDATE})
 148     endif()
 149   endif()
 150
 151   # Download
 152   if(NOT EXISTS "${CACHE_CANDIDATE}")
 153     ocv_download_log("#cmake_download \"${CACHE_CANDIDATE}\" \"${DL_URL}\"")
 154     file(DOWNLOAD "${DL_URL}" "${CACHE_CANDIDATE}"
 155          INACTIVITY_TIMEOUT 60
 156          TIMEOUT 600
 157          STATUS status
 158          LOG __log)
 159     if(NOT OPENCV_SKIP_FILE_DOWNLOAD_DUMP)  # workaround problem with old CMake versions: "Invalid escape sequence"
 160       string(LENGTH "${__log}" __log_length)
 161       if(__log_length LESS 65536)
 162         string(REPLACE "\n" "\n# " __log "${__log}")
 163         ocv_download_log("# ${__log}\n")
 164       endif()
 165     endif()
 166     if(NOT status EQUAL 0)
 167       set(msg_level FATAL_ERROR)
 168       if(DEFINED DL_STATUS)
 169         set(${DL_STATUS} FALSE PARENT_SCOPE)
 170         set(msg_level WARNING)
 171       endif()
 172       if(status MATCHES "Couldn't resolve host name")
 173         message(STATUS "
 174 =======================================================================
 175   Couldn't download files from the Internet.
 176   Please check the Internet access on this host.
 177 =======================================================================
 178 ")
 179       elseif(status MATCHES "Couldn't connect to server")
 180         message(STATUS "
 181 =======================================================================
 182   Couldn't connect to server from the Internet.
 183   Perhaps direct connections are not allowed in the current network.
 184   To use proxy please check/specify these environment variables:
 185   - http_proxy/https_proxy
 186   - and/or HTTP_PROXY/HTTPS_PROXY
 187 =======================================================================
 188 ")
 189       endif()
 190       message(${msg_level} "${__msg_prefix}Download failed: ${status}
 191 For details please refer to the download log file:
 192 ${OPENCV_DOWNLOAD_LOG}
 193 ")
 194       return()
 195     endif()
 196
 197     # Don't remove this code, because EXPECTED_MD5 parameter doesn't fail "file(DOWNLOAD)" step on wrong hash
 198     ocv_download_log("#check_md5 \"${CACHE_CANDIDATE}\"")
 199     file(MD5 "${CACHE_CANDIDATE}" target_md5)
 200     if(NOT target_md5 STREQUAL DL_HASH)
 201       ocv_download_log("#mismatch_md5 \"${CACHE_CANDIDATE}\" \"${target_md5}\"")
 202       set(msg_level FATAL_ERROR)
 203       if(DEFINED DL_STATUS)
 204         set(${DL_STATUS} FALSE PARENT_SCOPE)
 205         set(msg_level WARNING)
 206       endif()
 207       message(${msg_level} "${__msg_prefix}Hash mismatch: ${target_md5}")
 208       return()
 209     endif()
 210   endif()
 211
 212   # Unpack or copy
 213   if(DL_UNPACK)
 214     if(EXISTS "${DL_DESTINATION_DIR}")
 215       ocv_download_log("#remove_unpack \"${DL_DESTINATION_DIR}\"")
 216       file(REMOVE_RECURSE "${DL_DESTINATION_DIR}")
 217     endif()
 218     ocv_download_log("#mkdir \"${DL_DESTINATION_DIR}\"")
 219     file(MAKE_DIRECTORY "${DL_DESTINATION_DIR}")
 220     ocv_download_log("#unpack \"${DL_DESTINATION_DIR}\" \"${CACHE_CANDIDATE}\"")
 221     execute_process(COMMAND "${CMAKE_COMMAND}" -E tar xzf "${CACHE_CANDIDATE}"
 222                     WORKING_DIRECTORY "${DL_DESTINATION_DIR}"
 223                     RESULT_VARIABLE res)
 224     if(NOT res EQUAL 0)
 225       message(FATAL_ERROR "${__msg_prefix}Unpack failed: ${res}")
 226     endif()
 227   else()
 228     ocv_download_log("#copy \"${COPY_DESTINATION}\" \"${CACHE_CANDIDATE}\"")
 229     execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CACHE_CANDIDATE}" "${COPY_DESTINATION}"
 230                     RESULT_VARIABLE res)
 231     if(NOT res EQUAL 0)
 232       message(FATAL_ERROR "${__msg_prefix}Copy failed: ${res}")
 233     endif()
 234   endif()
 235
 236   if(OCV_DOWNLOAD_HASH_NAME)
 237     set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "")
 238   endif()
 239 endfunction()