From: David Steele Date: Tue, 24 Jul 2018 19:25:22 +0000 (+0100) Subject: Added base64 encoding/decoding for Property Array/String X-Git-Tag: dali_1.3.34~2 X-Git-Url: http://review.tizen.org/git/?p=platform%2Fcore%2Fuifw%2Fdali-toolkit.git;a=commitdiff_plain;h=be89a128154f4eb35ed88bbd03d446ab7c41cf64 Added base64 encoding/decoding for Property Array/String We need a means of reading and writing data blocks through the property system for CustomShaders in VK. Added a mechanism to allow encoding a vector of uin32_t data into a Property::Value of type STRING or ARRAY of STRINGS, and a means of decoding the same. This allows us to keep the property system in line with a JSON encoding, but trading off data size and decode time. Change-Id: I2247b340571644b21be5b9ef87fc0e3108e2f571 Signed-off-by: David Steele --- diff --git a/automated-tests/src/dali-toolkit/utc-Dali-Builder.cpp b/automated-tests/src/dali-toolkit/utc-Dali-Builder.cpp index 511d2ef..77439d4 100644 --- a/automated-tests/src/dali-toolkit/utc-Dali-Builder.cpp +++ b/automated-tests/src/dali-toolkit/utc-Dali-Builder.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Samsung Electronics Co., Ltd. + * Copyright (c) 2018 Samsung Electronics Co., Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,20 @@ */ #include -#include +#include +#include +#include +#include + #include #include +#include #include #include #include #include + #define STRINGIFY(A)#A using namespace Dali; @@ -1851,3 +1857,256 @@ int UtcDaliBuilderConfigurationP(void) END_TEST; } + + +int UtcDaliBase64EncodingP(void) +{ + std::vector data = { 0, 1, 2, 3, 4, 5, std::numeric_limits::min(), std::numeric_limits::max() }; + + Property::Value value; + EncodeBase64PropertyData( value, data ); + + std::cout << "Max uint32_t:" << std::numeric_limits::max() << std::endl; + std::cout << "Input data: "; + std::ostream_iterator out_it (std::cout,", "); + std::copy ( data.begin(), data.end(), out_it ); + std::cout << std::endl; + + std::string output; + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output, "AAAAAAEAAAACAAAAAwAAAAQAAAAFAAAAAAAAAP////8", TEST_LOCATION ); + + std::cout << "Output data: " << output << std::endl; + + END_TEST; +} + +int UtcDaliBase64EncodingN(void) +{ + tet_infoline( "Test encoding an empty vector returns empty string" ); + std::vector data; + + Property::Value value; + EncodeBase64PropertyData( value, data ); + + std::string output; + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output.empty(), true, TEST_LOCATION ); + + END_TEST; +} + +template +int b64l(std::vector&data) +{ + auto lengthInBytes = 4*data.size(); + return ceil( lengthInBytes * 1.33333f ); +} + +int UtcDaliBase64EncodingP02(void) +{ + tet_infoline( "Test encoding vectors of lengths m .. m+4 encode and decode back to the same length vectors" ); + + std::vector testData; + for(int i=0; i<8; ++i ) // 8 chosen to stay within single string output + { + testData.push_back(i); + } + Property::Value value; + EncodeBase64PropertyData( value, testData ); + + std::string output; + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output.empty(), false, TEST_LOCATION); + DALI_TEST_EQUALS( output.length(), b64l(testData), TEST_LOCATION ); + + std::vector outData; + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + DALI_TEST_EQUALS( std::equal( testData.begin(), testData.end(), outData.begin()), true, TEST_LOCATION ); + + // n+1 + testData.push_back( 12345 ); + EncodeBase64PropertyData( value, testData ); + + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output.empty(), false, TEST_LOCATION); + DALI_TEST_EQUALS( output.length(), b64l(testData), TEST_LOCATION ); + + outData.clear(); + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + DALI_TEST_EQUALS( std::equal( testData.begin(), testData.end(), outData.begin()), true, TEST_LOCATION ); + + // n+2 + testData.push_back( 67890 ); + EncodeBase64PropertyData( value, testData ); + + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output.empty(), false, TEST_LOCATION); + DALI_TEST_EQUALS( output.length(), b64l(testData), TEST_LOCATION ); + + outData.clear(); + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + DALI_TEST_EQUALS( std::equal( testData.begin(), testData.end(), outData.begin()), true, TEST_LOCATION ); + + // n+3 + testData.push_back( -1 ); + EncodeBase64PropertyData( value, testData ); + + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output.empty(), false, TEST_LOCATION); + DALI_TEST_EQUALS( output.length(), b64l(testData), TEST_LOCATION ); + + outData.clear(); + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + DALI_TEST_EQUALS( std::equal( testData.begin(), testData.end(), outData.begin()), true, TEST_LOCATION ); + + + END_TEST; +} + + +int UtcDaliBase64EncodingP03(void) +{ + tet_infoline( "Test encoding a vector of length 12 has output within single string" ); + + std::vector testData; + for(int i=0; i<12; ++i ) + { + testData.push_back(i); + } + Property::Value value; + EncodeBase64PropertyData( value, testData ); + + std::string output; + DALI_TEST_CHECK( value.Get( output ) ); + DALI_TEST_EQUALS( output.empty(), false, TEST_LOCATION); + DALI_TEST_EQUALS( output.length(), b64l(testData), TEST_LOCATION ); + + std::vector outData; + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + + END_TEST; +} + + +int UtcDaliBase64EncodingP04(void) +{ + tet_infoline( "Test encoding a vector of length 13 has output split over 2 strings" ); + + std::vector testData; + for(int i=0; i<13; ++i ) + { + testData.push_back(i); + } + Property::Value value; + EncodeBase64PropertyData( value, testData ); + + auto array = value.GetArray(); + DALI_TEST_CHECK( array ); + + DALI_TEST_EQUALS( array->Count(), 2, TEST_LOCATION ); + + std::vector outData; + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + + END_TEST; +} + + +int UtcDaliBase64EncodingP05(void) +{ + tet_infoline( "Test encoding a vector of length 24 has output split over 2 strings" ); + + std::vector testData; + for(int i=0; i<24; ++i ) + { + testData.push_back(i); + } + Property::Value value; + EncodeBase64PropertyData( value, testData ); + + auto array = value.GetArray(); + DALI_TEST_CHECK( array ); + + DALI_TEST_EQUALS( array->Count(), 2, TEST_LOCATION ); + + std::vector outData; + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + + END_TEST; +} + + +int UtcDaliBase64EncodingP06(void) +{ + tet_infoline( "Test encoding a vector of arbitrary length decodes OK." ); + + std::vector testData; + for(int i=0; i<97; ++i ) + { + testData.push_back(i); + } + Property::Value value; + EncodeBase64PropertyData( value, testData ); + + auto array = value.GetArray(); + DALI_TEST_CHECK( array ); + + std::vector outData; + DecodeBase64PropertyData( value, outData ); + DALI_TEST_EQUALS( testData.size(), outData.size(), TEST_LOCATION ); + + END_TEST; +} + + +int UtcDaliBase64DecodingN01(void) +{ + tet_infoline( "Test decoding empty string results in empty data" ); + + Property::Value value(""); + std::vector outputData; + DecodeBase64PropertyData( value, outputData); + DALI_TEST_EQUALS( outputData.size(), 0, TEST_LOCATION ); + END_TEST; +} + + +int UtcDaliBase64DecodingN02(void) +{ + tet_infoline( "Test decoding array with non-string values results in empty data" ); + + Property::Array array; + array.Resize(2); + array[0] = "Stuff, things"; + array[1] = 1; + Property::Value value(array); + + std::vector outputData; + DecodeBase64PropertyData( value, outputData); + DALI_TEST_EQUALS( outputData.size(), 0, TEST_LOCATION ); + END_TEST; +} + +int UtcDaliBase64DecodingP01(void) +{ + tet_infoline( "Test decoding string of known data gives expected result"); + + std::string testInput("//////7+/v4DAgEA"); + std::vector expectedResults = { 0xffffffff, 0xfefefefe, 0x00010203 }; + + std::vector outputData; + DecodeBase64PropertyData(Property::Value(testInput), outputData); + + DALI_TEST_EQUALS( std::equal( expectedResults.begin(), expectedResults.end(), outputData.begin() ), true, + TEST_LOCATION ); + + END_TEST; +} diff --git a/dali-toolkit/devel-api/builder/base64-encoding.cpp b/dali-toolkit/devel-api/builder/base64-encoding.cpp new file mode 100644 index 0000000..2799430 --- /dev/null +++ b/dali-toolkit/devel-api/builder/base64-encoding.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// EXTERNAL INCLUDES +#include +#include + +#include +#include +#include +#include + +namespace Dali +{ + +namespace Toolkit +{ + +namespace +{ +const int MAX_PROPERTY_STRING_LENGTH(64); // Cuts larger strings into blocks of this size. + +bool GetStringFromProperty( const Property::Value& value, std::string& output ) +{ + bool extracted = false; + if( value.Get( output ) ) + { + extracted = true; + } + else + { + Property::Array* array = value.GetArray(); + if( array ) + { + const unsigned int arraySize = array->Size(); + for( unsigned int i = 0; i < arraySize; ++i ) + { + std::string element; + if( array->GetElementAt( i ).Get( element ) ) + { + extracted = true; + output += element; + } + else + { + // If property in array is anything other than a string, then it is invalid so break and clear output. + output.clear(); + extracted = false; + break; + } + } + } + } + return extracted; +} + +}//anonymous namespace + +bool DecodeBase64PropertyData( const Property::Value& value, std::vector& outputData ) +{ + bool decoded = false; + std::string encodedString; + + + if( GetStringFromProperty( value, encodedString ) ) + { + std::vector outputTmpData; + outputTmpData.reserve( ceil( encodedString.size() * 0.75f ) ); + bn::decode_b64( encodedString.begin(), encodedString.end(), std::back_inserter(outputTmpData) ); + + outputData.clear(); + outputData.resize( outputTmpData.size() / sizeof(uint32_t) ); + // Treat as a block of data + memcpy( &outputData[0], &outputTmpData[0], outputTmpData.size() ); + + decoded = true; + } + return decoded; +} + +void EncodeBase64PropertyData( Property::Value& value, const std::vector& inputData ) +{ + std::ostringstream oss; + + bn::encode_b64( reinterpret_cast(&inputData[0]), + reinterpret_cast(&inputData[0]+inputData.size()), + std::ostream_iterator(oss, "") ); + + + std::string encodedString = oss.str(); + if( encodedString.length() > MAX_PROPERTY_STRING_LENGTH) + { + // cut string up into blocks of MAX_PROPERTY_STRING_LENGTH and store to an array + auto numStrings = encodedString.length() / MAX_PROPERTY_STRING_LENGTH + + ((encodedString.length() % MAX_PROPERTY_STRING_LENGTH) != 0); + + Property::Array array; + for(auto i=0u; i +#include + +namespace Dali +{ + +namespace Toolkit +{ + +/** + * @brief Parses a Property::STRING or Property::ARRAY of STRINGS to + * retrieve a block of uint32_t data. + * + * Data can be encoded using the base64 encoding scheme to allow it to be used + * in JSON (The property system maps to JSON types). + * + * @param[in] value The property value to decode + * @param[out] outputData The output data block + * @return True if a data block was decoded successfully. + */ +bool DecodeBase64PropertyData( const Property::Value& value, std::vector& outputData ); + +/** + * @brief Convert a block of uint32_t data into a Property::STRING or ARRAY of STRINGs + * encoded using base64. This allows the data to be mapped to JSON easily. + * + * @param[out] value The value to write data into (to avoid copying). + * @param[in] inputData The input + */ +void EncodeBase64PropertyData( Property::Value& value, const std::vector& inputData ); + +} // namespace Toolkit + +} // namespace Dali + + +#endif // DALI_TOOLKIT_BASE64_ENCODING_H diff --git a/dali-toolkit/devel-api/file.list b/dali-toolkit/devel-api/file.list index 411afe1..e974720 100755 --- a/dali-toolkit/devel-api/file.list +++ b/dali-toolkit/devel-api/file.list @@ -1,6 +1,7 @@ # Add local source files here devel_api_src_files = \ + $(devel_api_src_dir)/builder/base64-encoding.cpp \ $(devel_api_src_dir)/builder/builder.cpp \ $(devel_api_src_dir)/builder/json-parser.cpp \ $(devel_api_src_dir)/builder/tree-node.cpp \ @@ -73,6 +74,7 @@ devel_api_buttons_header_files = \ $(devel_api_src_dir)/controls/buttons/toggle-button.h devel_api_builder_header_files = \ + $(devel_api_src_dir)/builder/base64-encoding.h \ $(devel_api_src_dir)/builder/builder.h \ $(devel_api_src_dir)/builder/json-parser.h \ $(devel_api_src_dir)/builder/tree-node.h diff --git a/dali-toolkit/third-party/base-n/basen.hpp b/dali-toolkit/third-party/base-n/basen.hpp new file mode 100644 index 0000000..3ed31e8 --- /dev/null +++ b/dali-toolkit/third-party/base-n/basen.hpp @@ -0,0 +1,289 @@ +/** + * base-n, 1.0 + * Copyright (C) 2012 Andrzej Zawadzki (azawadzki@gmail.com) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. +**/ +#ifndef BASEN_HPP +#define BASEN_HPP + +#include +#include +#include +#include + +namespace bn +{ + +template +void encode_b16(Iter1 start, Iter1 end, Iter2 out); + +template +void encode_b32(Iter1 start, Iter1 end, Iter2 out); + +template +void encode_b64(Iter1 start, Iter1 end, Iter2 out); + +template +void decode_b16(Iter1 start, Iter1 end, Iter2 out); + +template +void decode_b32(Iter1 start, Iter1 end, Iter2 out); + +template +void decode_b64(Iter1 start, Iter1 end, Iter2 out); + +namespace impl +{ + +const int Error = -1; + +namespace { + +char extract_partial_bits(char value, size_t start_bit, size_t bits_count) +{ + assert(start_bit + bits_count < 8); + // shift extracted bits to the beginning of the byte + char t1 = value >> (8 - bits_count - start_bit); + // mask out bits on the left + char t2 = t1 & ~(0xff << bits_count); + return t2; +} + +char extract_overlapping_bits(char previous, char next, size_t start_bit, size_t bits_count) +{ + assert(start_bit + bits_count < 16); + size_t bits_count_in_previous = 8 - start_bit; + size_t bits_count_in_next = bits_count - bits_count_in_previous; + char t1 = previous << bits_count_in_next; + char t2 = next >> (8 - bits_count_in_next) & ~(0xff << bits_count_in_next) ; + return (t1 | t2) & ~(0xff << bits_count); +} + +} + +struct b16_conversion_traits +{ + static size_t group_length() + { + return 4; + } + + static char encode(unsigned int index) + { + const char* const dictionary = "0123456789ABCDEF"; + assert(index < strlen(dictionary)); + return dictionary[index]; + } + + static char decode(char c) + { + if (c >= '0' && c <= '9') { + return c - '0'; + } else if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return Error; + } +}; + +struct b32_conversion_traits +{ + static size_t group_length() + { + return 5; + } + + static char encode(unsigned int index) + { + const char * dictionary = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + assert(index < strlen(dictionary)); + return dictionary[index]; + } + + static char decode(char c) + { + if (c >= 'A' && c <= 'Z') { + return c - 'A'; + } else if (c >= '2' && c <= '7') { + return c - '2' + 26; + } + return Error; + } +}; + +struct b64_conversion_traits +{ + static size_t group_length() + { + return 6; + } + + static char encode(unsigned int index) + { + const char* const dictionary = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + assert(index < strlen(dictionary)); + return dictionary[index]; + } + + static char decode(char c) + { + const int alph_len = 26; + if (c >= 'A' && c <= 'Z') { + return c - 'A'; + } else if (c >= 'a' && c <= 'z') { + return c - 'a' + alph_len * 1; + } else if (c >= '0' && c <= '9') { + return c - '0' + alph_len * 2; + } else if (c == '+') { + return c - '+' + alph_len * 2 + 10; + } else if (c == '/') { + return c - '/' + alph_len * 2 + 11; + } + return Error; + } +}; + +template +void decode(Iter1 start, Iter1 end, Iter2 out) +{ + Iter1 iter = start; + size_t output_current_bit = 0; + char buffer = 0; + + while (iter != end) { + if (std::isspace(*iter)) { + ++iter; + continue; + } + char value = ConversionTraits::decode(*iter); + if (value == Error) { + // malformed data, but let's go on... + ++iter; + continue; + } + size_t bits_in_current_byte = std::min(output_current_bit + ConversionTraits::group_length(), 8) - output_current_bit; + if (bits_in_current_byte == ConversionTraits::group_length()) { + // the value fits within current byte, so we can extract it directly + buffer |= value << (8 - output_current_bit - ConversionTraits::group_length()); + output_current_bit += ConversionTraits::group_length(); + // check if we filled up current byte completely; in such case we flush output and continue + if (output_current_bit == 8) { + *out++ = buffer; + buffer = 0; + output_current_bit = 0; + } + } else { + // the value spans across the current and the next byte + size_t bits_in_next_byte = ConversionTraits::group_length() - bits_in_current_byte; + // fill the current byte and flush it to our output + buffer |= value >> bits_in_next_byte; + *out++ = buffer; + buffer = 0; + // save the remainder of our value in the buffer; it will be flushed + // during next iterations + buffer |= value << (8 - bits_in_next_byte); + output_current_bit = bits_in_next_byte; + } + ++iter; + } +} + +template +void encode(Iter1 start, Iter1 end, Iter2 out) +{ + Iter1 iter = start; + size_t start_bit = 0; + bool has_backlog = false; + char backlog = 0; + + while (has_backlog || iter != end) { + if (!has_backlog) { + if (start_bit + ConversionTraits::group_length() < 8) { + // the value fits within single byte, so we can extract it + // directly + char v = extract_partial_bits(*iter, start_bit, ConversionTraits::group_length()); + *out++ = ConversionTraits::encode(v); + // since we know that start_bit + ConversionTraits::group_length() < 8 we don't need to go + // to the next byte + start_bit += ConversionTraits::group_length(); + } else { + // our bits are spanning across byte border; we need to keep the + // starting point and move over to next byte. + backlog = *iter++; + has_backlog = true; + } + } else { + // encode value which is made from bits spanning across byte + // boundary + char v; + if (iter == end) + v = extract_overlapping_bits(backlog, 0, start_bit, ConversionTraits::group_length()); + else + v = extract_overlapping_bits(backlog, *iter, start_bit, ConversionTraits::group_length()); + *out++ = ConversionTraits::encode(v); + has_backlog = false; + start_bit = (start_bit + ConversionTraits::group_length()) % 8; + } + } +} + +} // impl + +using namespace bn::impl; + +template +void encode_b16(Iter1 start, Iter1 end, Iter2 out) +{ + encode(start, end, out); +} + +template +void encode_b32(Iter1 start, Iter1 end, Iter2 out) +{ + encode(start, end, out); +} + +template +void encode_b64(Iter1 start, Iter1 end, Iter2 out) +{ + encode(start, end, out); +} + +template +void decode_b16(Iter1 start, Iter1 end, Iter2 out) +{ + decode(start, end, out); +} + +template +void decode_b32(Iter1 start, Iter1 end, Iter2 out) +{ + decode(start, end, out); +} + +template +void decode_b64(Iter1 start, Iter1 end, Iter2 out) +{ + decode(start, end, out); +} + +} // bn + +#endif // BASEN_HPP