Changed curl file download algorithm to handle chunks

Some websites don't provide an image size in the HTTP header, instead
they 'chunk' the data. The new implementation handles both images with
a size that can be allocated, and images that are loaded as chunks.

Change-Id: If8c4b914f75581f093b51c92e2d3e282b5e9dd5b
Signed-off-by: David Steele <david.steele@samsung.com>

diff --git a/automated-tests/src/dali-adaptor/utc-Dali-ImageLoading.cpp b/automated-tests/src/dali-adaptor/utc-Dali-ImageLoading.cpp
index b05ffed..8d09fac 100644 (file)
--- a/automated-tests/src/dali-adaptor/utc-Dali-ImageLoading.cpp
+++ b/automated-tests/src/dali-adaptor/utc-Dali-ImageLoading.cpp
@@ -99,3 +99,17 @@ int UtcDaliDownloadImageN(void)
 
   END_TEST;
 }
+
+
+int UtcDaliDownloadRemoteChunkedImage(void)
+{
+  std::string url("http://d2k43l0oslhof9.cloudfront.net/platform/image/contents/vc/20/01/58/20170629100630071189_0bf6b911-a847-cba4-e518-be40fe2f579420170629192203240.jpg");
+
+  Devel::PixelBuffer pixelBuffer = Dali::DownloadImageSynchronously( url );
+  DALI_TEST_CHECK( pixelBuffer );
+  DALI_TEST_EQUALS( pixelBuffer.GetWidth(), 279u, TEST_LOCATION );
+  DALI_TEST_EQUALS( pixelBuffer.GetHeight(), 156u, TEST_LOCATION );
+  DALI_TEST_EQUALS( pixelBuffer.GetPixelFormat(), Pixel::RGBA8888, TEST_LOCATION  );
+
+  END_TEST;
+}

diff --git a/platform-abstractions/tizen/resource-loader/network/file-download.cpp b/platform-abstractions/tizen/resource-loader/network/file-download.cpp
index 1e33dc3..ad50249 100755 (executable)
--- a/platform-abstractions/tizen/resource-loader/network/file-download.cpp
+++ b/platform-abstractions/tizen/resource-loader/network/file-download.cpp
@@ -21,6 +21,7 @@
 // EXTERNAL INCLUDES
 #include <dali/integration-api/debug.h>
 #include <curl/curl.h>
+#include <cstring>
 
 // INTERNAL INCLUDES
 #include "portable/file-closer.h"
@@ -55,20 +56,20 @@ const long EXCLUDE_BODY = 1L;
 static Dali::TizenPlatform::Network::CurlEnvironment gCurlEnvironment;
 
 
-void ConfigureCurlOptions( CURL* curl_handle, const std::string& url )
+void ConfigureCurlOptions( CURL* curlHandle, const std::string& url )
 {
-  curl_easy_setopt( curl_handle, CURLOPT_URL, url.c_str() );
-  curl_easy_setopt( curl_handle, CURLOPT_VERBOSE, VERBOSE_MODE );
+  curl_easy_setopt( curlHandle, CURLOPT_URL, url.c_str() );
+  curl_easy_setopt( curlHandle, CURLOPT_VERBOSE, VERBOSE_MODE );
 
   // CURLOPT_FAILONERROR is not fail-safe especially when authentication is involved ( see manual )
-  curl_easy_setopt( curl_handle, CURLOPT_FAILONERROR, CLOSE_CONNECTION_ON_ERROR );
-  curl_easy_setopt( curl_handle, CURLOPT_CONNECTTIMEOUT, CONNECTION_TIMEOUT_SECONDS );
-  curl_easy_setopt( curl_handle, CURLOPT_HEADER, INCLUDE_HEADER );
-  curl_easy_setopt( curl_handle, CURLOPT_NOBODY, EXCLUDE_BODY );
+  curl_easy_setopt( curlHandle, CURLOPT_FAILONERROR, CLOSE_CONNECTION_ON_ERROR );
+  curl_easy_setopt( curlHandle, CURLOPT_CONNECTTIMEOUT, CONNECTION_TIMEOUT_SECONDS );
+  curl_easy_setopt( curlHandle, CURLOPT_HEADER, INCLUDE_HEADER );
+  curl_easy_setopt( curlHandle, CURLOPT_NOBODY, EXCLUDE_BODY );
 
 #ifdef TPK_CURL_ENABLED
   // Apply certificate pinning on Tizen
-  curl_easy_setopt( curl_handle, CURLOPT_SSL_CTX_FUNCTION, tpkp_curl_ssl_ctx_callback );
+  curl_easy_setopt( curlHandle, CURLOPT_SSL_CTX_FUNCTION, tpkp_curl_ssl_ctx_callback );
 #endif // TPK_CURL_ENABLED
 }
 
@@ -79,69 +80,127 @@ size_t DummyWrite(char *ptr, size_t size, size_t nmemb, void *userdata)
   return size * nmemb;
 }
 
+struct ChunkData
+{
+  std::vector< uint8_t > data;
+};
+
+size_t ChunkLoader(char *ptr, size_t size, size_t nmemb, void *userdata)
+{
+  std::vector<ChunkData>* chunks = static_cast<std::vector<ChunkData>*>( userdata );
+  int numBytes = size*nmemb;
+  chunks->push_back( ChunkData() );
+  ChunkData& chunkData = (*chunks)[chunks->size()-1];
+  chunkData.data.reserve( numBytes );
+  memcpy( &chunkData.data[0], ptr, numBytes );
+  return numBytes;
+}
+
+
+CURLcode DownloadFileDataWithSize( CURL* curlHandle, Dali::Vector<uint8_t>& dataBuffer, size_t dataSize )
+{
+  CURLcode result( CURLE_OK );
+  dataBuffer.Resize( dataSize );
+
+  // create
+  Dali::Internal::Platform::FileCloser fileCloser( static_cast<void*>(&dataBuffer[0]), dataSize, "wb" );
+  FILE* dataBufferFilePointer = fileCloser.GetFile();
+  if( NULL != dataBufferFilePointer )
+  {
+    // we only want the body which contains the file data
+    curl_easy_setopt( curlHandle, CURLOPT_HEADER, EXCLUDE_HEADER );
+    curl_easy_setopt( curlHandle, CURLOPT_NOBODY, INCLUDE_BODY );
 
-bool DownloadFile( CURL* curl_handle,
+    // disable the write callback, and get curl to write directly into our data buffer
+    curl_easy_setopt( curlHandle, CURLOPT_WRITEFUNCTION, NULL );
+    curl_easy_setopt( curlHandle, CURLOPT_WRITEDATA, dataBufferFilePointer );
+
+    // synchronous request of the body data
+    result = curl_easy_perform( curlHandle );
+  }
+  return result;
+}
+
+CURLcode DownloadFileDataByChunk( CURL* curlHandle, Dali::Vector<uint8_t>& dataBuffer, size_t& dataSize )
+{
+  // create
+  std::vector< ChunkData > chunks;
+
+  // we only want the body which contains the file data
+  curl_easy_setopt( curlHandle, CURLOPT_HEADER, EXCLUDE_HEADER );
+  curl_easy_setopt( curlHandle, CURLOPT_NOBODY, INCLUDE_BODY );
+
+  // Enable the write callback.
+  curl_easy_setopt( curlHandle, CURLOPT_WRITEFUNCTION, ChunkLoader );
+  curl_easy_setopt( curlHandle, CURLOPT_WRITEDATA, &chunks );
+
+  // synchronous request of the body data
+  CURLcode result = curl_easy_perform( curlHandle );
+
+  // chunks should now contain all of the chunked data. Reassemble into a single vector
+  dataSize = 0;
+  for( size_t i=0; i<chunks.size() ; ++i )
+  {
+    dataSize += chunks[i].data.capacity();
+  }
+  dataBuffer.Resize(dataSize);
+
+  size_t offset = 0;
+  for( size_t i=0; i<chunks.size() ; ++i )
+  {
+    memcpy( &dataBuffer[offset], &chunks[i].data[0], chunks[i].data.capacity() );
+    offset += chunks[i].data.capacity();
+  }
+
+  return result;
+}
+
+bool DownloadFile( CURL* curlHandle,
                    const std::string& url,
                    Dali::Vector<uint8_t>& dataBuffer,
                    size_t& dataSize,
                    size_t maximumAllowedSizeBytes )
 {
-  CURLcode res( CURLE_OK );
+  CURLcode result( CURLE_OK );
   double size(0);
 
   // setup curl to download just the header so we can extract the content length
-  ConfigureCurlOptions( curl_handle, url );
+  ConfigureCurlOptions( curlHandle, url );
 
-  curl_easy_setopt( curl_handle, CURLOPT_WRITEFUNCTION, DummyWrite);
+  curl_easy_setopt( curlHandle, CURLOPT_WRITEFUNCTION, DummyWrite);
 
   // perform the request to get the header
-  res = curl_easy_perform( curl_handle );
+  result = curl_easy_perform( curlHandle );
 
-  if( res != CURLE_OK)
+  if( result != CURLE_OK)
   {
-    DALI_LOG_WARNING( "Failed to download http header for \"%s\" with error code %d\n", url.c_str(), res );
+    DALI_LOG_WARNING( "Failed to download http header for \"%s\" with error code %d\n", url.c_str(), result );
     return false;
   }
 
   // get the content length, -1 == size is not known
-  curl_easy_getinfo( curl_handle,CURLINFO_CONTENT_LENGTH_DOWNLOAD , &size );
+  curl_easy_getinfo( curlHandle,CURLINFO_CONTENT_LENGTH_DOWNLOAD , &size );
+
 
-  if( size < 1 )
-  {
-    DALI_LOG_WARNING( "Header missing content length \"%s\" \n", url.c_str() );
-    return false;
-  }
   if( size >= maximumAllowedSizeBytes )
   {
     DALI_LOG_WARNING( "File content length %f > max allowed %zu \"%s\" \n", size, maximumAllowedSizeBytes, url.c_str() );
     return false;
   }
-
-  dataSize = static_cast<size_t>( size );
-
-  dataBuffer.Resize( dataSize );
-
-  // create
-  Dali::Internal::Platform::FileCloser fileCloser( static_cast<void*>(&dataBuffer[0]), dataSize, "wb" );
-  FILE* dataBufferFilePointer = fileCloser.GetFile();
-  if( NULL != dataBufferFilePointer )
+  else if( size > 0 )
   {
-    // we only want the body which contains the file data
-    curl_easy_setopt( curl_handle, CURLOPT_HEADER, EXCLUDE_HEADER );
-    curl_easy_setopt( curl_handle, CURLOPT_NOBODY, INCLUDE_BODY );
-
-    // disable the write callback, and get curl to write directly into our data buffer
-    curl_easy_setopt( curl_handle, CURLOPT_WRITEFUNCTION, NULL );
-    curl_easy_setopt( curl_handle, CURLOPT_WRITEDATA, dataBufferFilePointer );
-
-    // synchronous request of the body data
-    res = curl_easy_perform( curl_handle );
+    dataSize = static_cast<size_t>( size );
+    result = DownloadFileDataWithSize( curlHandle, dataBuffer, dataSize );
+  }
+  else
+  {
+    result = DownloadFileDataByChunk( curlHandle, dataBuffer, dataSize );
+  }
 
-    if( CURLE_OK != res )
-    {
-      DALI_LOG_WARNING( "Failed to download image file \"%s\" with error code %d\n", url.c_str(), res );
-      return false;
-    }
+  if( result != CURLE_OK )
+  {
+    DALI_LOG_WARNING( "Failed to download image file \"%s\" with error code %d\n", url.c_str(), result );
+    return false;
   }
   return true;
 }
@@ -180,12 +239,12 @@ bool DownloadRemoteFileIntoMemory( const std::string& url,
   // start a libcurl easy session, this internally calls curl_global_init, if we ever have more than one download
   // thread we need to explicity call curl_global_init() on startup from a single thread.
 
-  CURL* curl_handle = curl_easy_init();
+  CURL* curlHandle = curl_easy_init();
 
-  bool result = DownloadFile( curl_handle, url, dataBuffer,  dataSize, maximumAllowedSizeBytes);
+  bool result = DownloadFile( curlHandle, url, dataBuffer,  dataSize, maximumAllowedSizeBytes);
 
   // clean up session
-  curl_easy_cleanup( curl_handle );
+  curl_easy_cleanup( curlHandle );
 
 #ifdef TPK_CURL_ENABLED
   // Clean up tpkp(the module for certificate pinning) resources on Tizen