Replaced buggy and overcomplicated base64 functions with simpler and more efficient implementation (f53de445) · Commits · TNL / tnl-dev

src/TNL/Meshes/Readers/XMLVTK.h

+3 −3

src/TNL/Meshes/Writers/VTUWriter.hpp

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -504,7 +504,7 @@ VTUWriter< Mesh >::writeDataArray( const Array& array,
	#endif		#endif
	// fall through to binary if HAVE_ZLIB is not defined		// fall through to binary if HAVE_ZLIB is not defined
	case VTK::FileFormat::binary:		case VTK::FileFormat::binary:
	write_encoded_block< HeaderType >( array.getData(), array.getSize(), str );		base64::write_encoded_block< HeaderType >( array.getData(), array.getSize(), str );
	str << "\n";		str << "\n";
	break;		break;
	}		}

src/TNL/base64.h

+158 −259

Original line number	Original line	Diff line number	Diff line
	@@ -11,280 +11,192 @@
	#pragma once		#pragma once

	#include <cstddef>		#include <cstddef>
			#include <cstdint>
	#include <memory>		#include <memory>
	#include <utility>		#include <utility>
			#include <stdexcept>
	#include <cmath> // std::ceil		#include <cmath> // std::ceil

	namespace TNL {		namespace TNL {
			/**
	// The functions in the base64 namespace are taken from the libb64 project, see		* \brief Namespace for base64 encoding and decoding functions.
	// http://sourceforge.net/projects/libb64		*
	//		* The actual algorithms are based on these sources:
	// libb64 has been placed in the public domain		*
			* - http://web.mit.edu/freebsd/head/contrib/wpa/src/utils/base64.c
			* - https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c/
			* - https://stackoverflow.com/questions/342409/how-do-i-base64-encode-decode-in-c
			*/
	namespace base64 {		namespace base64 {

	// encoding		/**
			* \brief Get the length of base64-encoded block for given data byte length.
	typedef enum		*/
	{		inline std::size_t
	step_A,		get_encoded_length( std::size_t byte_length )
	step_B,
	step_C
	} base64_encodestep;

	typedef struct
	{
	base64_encodestep step;
	char result;
	} base64_encodestate;

	inline void
	base64_init_encodestate(base64_encodestate *state_in)
	{
	state_in->step = step_A;
	state_in->result = 0;
	}

	inline char
	base64_encode_value(char value_in)
	{		{
	static const char *encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";		std::size_t encoded = std::ceil(byte_length * (4.0 / 3.0));
	if (value_in > 63)		// base64 uses padding to a multiple of 4
	return '=';		if( encoded % 4 == 0 )
	return encoding[(int)value_in];		return encoded;
			return encoded + 4 - (encoded % 4);
	}		}

	inline std::ptrdiff_t		/**
	base64_encode_block(const char * plaintext_in,		* \brief Static table for base64 encoding.
	std::size_t length_in,		*/
	char * code_out,		static constexpr unsigned char encoding_table[65] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
	base64_encodestate *state_in)
	{
	const char * plainchar = plaintext_in;
	const char *const plaintextend = plaintext_in + length_in;
	char * codechar = code_out;
	char result;

	result = state_in->result;		/**
			* \brief Static table for base64 decoding.
			*
			* Can be built with the following code:
			*
			* \code
			* std::uint8_t decoding_table[256];
			* for( int i = 0; i < 256; i++ )
			* decoding_table[i] = 128;
			* for( std::uint8_t i = 0; i < sizeof(encoding_table) - 1; i++ )
			* decoding_table[encoding_table[i]] = i;
			* decoding_table[(int) '='] = 0;
			* \endcode
			*/
			static constexpr std::uint8_t decoding_table[256] = {
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 62, 128, 128, 128, 63,
			52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 128, 128, 128, 0, 128, 128,
			128, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
			15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 128, 128, 128, 128, 128,
			128, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
			41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
			128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
			};

	switch (state_in->step)		/**
	{		* \brief Do a base64 encoding of the given data.
	while (true)		*
	{		* \param data Pointer to the data to be encoded.
	case step_A:		* \param data_size Length of the input data (in bytes).
	{		* \return A \ref std::unique_ptr to the encoded data.
	if (plainchar == plaintextend)		*/
	{		inline std::unique_ptr<char[]>
	state_in->result = result;		encode( const std::uint8_t* data, std::size_t data_size )
	state_in->step = step_A;
	return codechar - code_out;
	}
	const char fragment = *plainchar++;
	result = (fragment & 0x0fc) >> 2;
	*codechar++ = base64_encode_value(result);
	result = (fragment & 0x003) << 4;
	// intended fallthrough
	}
	case step_B:
	{
	if (plainchar == plaintextend)
	{
	state_in->result = result;
	state_in->step = step_B;
	return codechar - code_out;
	}
	const char fragment = *plainchar++;
	result \|= (fragment & 0x0f0) >> 4;
	*codechar++ = base64_encode_value(result);
	result = (fragment & 0x00f) << 2;
	// intended fallthrough
	}
	case step_C:
	{
	if (plainchar == plaintextend)
	{		{
	state_in->result = result;		const std::size_t output_length = get_encoded_length( data_size );
	state_in->step = step_C;		std::unique_ptr<char[]> encoded_data{new char[output_length + 1]};
	return codechar - code_out;
	}
	const char fragment = *plainchar++;
	result \|= (fragment & 0x0c0) >> 6;
	*codechar++ = base64_encode_value(result);
	result = (fragment & 0x03f) >> 0;
	*codechar++ = base64_encode_value(result);
	}
	}
	}
	/* control should not reach here */
	return codechar - code_out;
	}

	inline std::ptrdiff_t		const std::uint8_t* end = data + data_size;
	base64_encode_blockend(char code_out, base64_encodestate state_in)		const std::uint8_t* in = data;
	{		char* out = encoded_data.get();
	char *codechar = code_out;		char* pos = out;

	switch (state_in->step)		while( end - in >= 3 ) {
	{		*pos++ = encoding_table[in[0] >> 2];
	case step_B:		*pos++ = encoding_table[((in[0] & 0x03) << 4) \| (in[1] >> 4)];
	*codechar++ = base64_encode_value(state_in->result);		*pos++ = encoding_table[((in[1] & 0x0f) << 2) \| (in[2] >> 6)];
	*codechar++ = '=';		*pos++ = encoding_table[in[2] & 0x3f];
	*codechar++ = '=';		in += 3;
	break;
	case step_C:
	*codechar++ = base64_encode_value(state_in->result);
	*codechar++ = '=';
	break;
	case step_A:
	break;
	}		}
	*codechar++ = '\0';

	return codechar - code_out;		if( end - in ) {
			*pos++ = encoding_table[in[0] >> 2];
			if( end - in == 1 ) {
			*pos++ = encoding_table[(in[0] & 0x03) << 4];
			*pos++ = '=';
	}		}
			else {
			*pos++ = encoding_table[((in[0] & 0x03) << 4) \| (in[1] >> 4)];
	// decoding		*pos++ = encoding_table[(in[1] & 0x0f) << 2];

	typedef enum
	{
	step_a, step_b, step_c, step_d
	} base64_decodestep;

	typedef struct
	{
	base64_decodestep step;
	char plainchar;
	} base64_decodestate;

	inline void
	base64_init_decodestate(base64_decodestate* state_in)
	{
	state_in->step = step_a;
	state_in->plainchar = 0;
	}		}
			*pos++ = '=';
	inline int
	base64_decode_value(char value_in)
	{
	static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51};
	static const char decoding_size = sizeof(decoding);
	value_in -= 43;
	if (value_in < 0 \|\| value_in >= decoding_size) return -1;
	return decoding[(int)value_in];
	}		}

	inline std::ptrdiff_t		*pos++ = '\0';
	base64_decode_block(const char* code_in, const std::size_t length_in, char* plaintext_out, base64_decodestate* state_in)		return encoded_data;
	{
	const char* codechar = code_in;
	char* plainchar = plaintext_out;
	char fragment;

	*plainchar = state_in->plainchar;

	switch (state_in->step)
	{
	while (1)
	{
	case step_a:
	do {
	if (codechar == code_in+length_in)
	{
	state_in->step = step_a;
	state_in->plainchar = *plainchar;
	return plainchar - plaintext_out;
	}
	fragment = (char)base64_decode_value(*codechar++);
	} while (fragment < 0);
	*plainchar = (fragment & 0x03f) << 2;
	case step_b:
	do {
	if (codechar == code_in+length_in)
	{
	state_in->step = step_b;
	state_in->plainchar = *plainchar;
	return plainchar - plaintext_out;
	}
	fragment = (char)base64_decode_value(*codechar++);
	} while (fragment < 0);
	*plainchar++ \|= (fragment & 0x030) >> 4;
	*plainchar = (fragment & 0x00f) << 4;
	case step_c:
	do {
	if (codechar == code_in+length_in)
	{
	state_in->step = step_c;
	state_in->plainchar = *plainchar;
	return plainchar - plaintext_out;
	}
	fragment = (char)base64_decode_value(*codechar++);
	} while (fragment < 0);
	*plainchar++ \|= (fragment & 0x03c) >> 2;
	*plainchar = (fragment & 0x003) << 6;
	case step_d:
	do {
	if (codechar == code_in+length_in)
	{
	state_in->step = step_d;
	state_in->plainchar = *plainchar;
	return plainchar - plaintext_out;
	}
	fragment = (char)base64_decode_value(*codechar++);
	} while (fragment < 0);
	*plainchar++ \|= (fragment & 0x03f);
	}
	}
	/* control should not reach here */
	return plainchar - plaintext_out;
	}		}

	} // namespace base64


	/**		/**
	* Do a base64 encoding of the given data.		* \brief Internal base64 decoding function.
	*		*
	* The function returns a unique_ptr to the encoded data.		* \param input Pointer to the encoded data (C string).
			* \param input_length Length of the input string.
			* \param output Pointer to a pre-allocated output buffer.
			* \param output_length Length of the output buffer.
			* \return Size of the decoded data (in bytes).
	*/		*/
	inline std::unique_ptr<char[]>		inline std::ptrdiff_t
	encode_block(const char* data, const std::size_t data_size)		decode_block( const char* input, std::size_t input_length, std::uint8_t* output, std::size_t output_length )
	{		{
	base64::base64_encodestate state;		const std::size_t min_buffer_size = std::ceil(input_length * (3.0 / 4.0));
	base64::base64_init_encodestate(&state);		if( output_length < min_buffer_size )
			throw std::logic_error( "base64: insufficient output buffer size " + std::to_string(output_length)
	std::unique_ptr<char[]> encoded_data{new char[2 * data_size + 1]};		+ " (needed at least " + std::to_string(min_buffer_size) + " bytes)" );

			std::size_t count = 0;
			int pad = 0;
			std::uint8_t block[4];
			std::uint8_t* pos = output;

			for (std::size_t i = 0; i < input_length; i++) {
			const std::uint8_t tmp = decoding_table[(int) input[i]];
			if( tmp == 128 )
			continue;

			if( input[i] == '=' )
			pad++;
			block[count] = tmp;
			count++;

			if( count == 4 ) {
			*pos++ = (block[0] << 2) \| (block[1] >> 4);
			*pos++ = (block[1] << 4) \| (block[2] >> 2);
			*pos++ = (block[2] << 6) \| block[3];
			count = 0;
			if( pad > 2 )
			// invalid padding
			throw std::invalid_argument( "base64: decoding error: input has invalid padding" );
			if( pad > 0 ) {
			pos -= pad;
			break;
			}
			}
			}

	const std::size_t encoded_length_data = base64::base64_encode_block(data, data_size, encoded_data.get(), &state);		// check left-over chars
	base64::base64_encode_blockend(encoded_data.get() + encoded_length_data, &state);		if( count )
			throw std::invalid_argument( "base64: decoding error: invalid input (length not padded to a multiple of 4)" );

	return encoded_data;		return pos - output;
	}		}


	/**		/**
	* Do a base64 decoding of the given data.		* \brief Do a base64 decoding of the given data.
	*		*
	* The function returns a pair of the decoded data length and a unique_ptr to		* \param data Pointer to the encoded data (C string).
	* the decoded data.		* \param data_size Length of the input string.
			* \return A pair of the decoded data length and a \ref std::unique_ptr to the
			* decoded data.
	*/		*/
	inline std::pair<std::size_t, std::unique_ptr<char[]>>		inline std::pair<std::size_t, std::unique_ptr<std::uint8_t[]>>
	decode_block(const char* data, const std::size_t data_size)		decode( const char* data, const std::size_t data_size )
	{		{
	base64::base64_decodestate state;		const std::size_t buffer_size = std::ceil(data_size * (3.0 / 4.0));
	base64::base64_init_decodestate(&state);		std::unique_ptr<std::uint8_t[]> decoded_data{new std::uint8_t[buffer_size + 1]};

	std::unique_ptr<char[]> decoded_data{new char[data_size + 1]};

	const std::size_t decoded_length_data = base64::base64_decode_block(data, data_size, decoded_data.get(), &state);
	decoded_data[decoded_length_data] = '\0';

			const std::size_t decoded_length_data = decode_block( data, data_size, decoded_data.get(), buffer_size );
	return {decoded_length_data, std::move(decoded_data)};		return {decoded_length_data, std::move(decoded_data)};
	}		}


	/**		/**
	* Write a base64-encoded block of data into the given stream.		* \brief Write a base64-encoded block of data into the given stream.
	*		*
	* The encoded data is prepended with a short header, which is the base64-encoded		* The encoded data is prepended with a short header, which is the base64-encoded
	* byte length of the data. The type of the byte length value is `HeaderType`.		* byte length of the data. The type of the byte length value is `HeaderType`.
	@@ -293,24 +205,11 @@ template <typename HeaderType = std::uint64_t, typename T>
	void write_encoded_block(const T* data, const std::size_t data_length, std::ostream& output_stream)		void write_encoded_block(const T* data, const std::size_t data_length, std::ostream& output_stream)
	{		{
	const HeaderType size = data_length * sizeof(T);		const HeaderType size = data_length * sizeof(T);
	std::unique_ptr<char[]> encoded_size = encode_block(reinterpret_cast<const char*>(&size), sizeof(HeaderType));		std::unique_ptr<char[]> encoded_size = base64::encode(reinterpret_cast<const std::uint8_t*>(&size), sizeof(HeaderType));
	output_stream << encoded_size.get();		output_stream << encoded_size.get();
	std::unique_ptr<char[]> encoded_data = encode_block(reinterpret_cast<const char*>(data), size);		std::unique_ptr<char[]> encoded_data = base64::encode(reinterpret_cast<const std::uint8_t*>(data), size);
	output_stream << encoded_data.get();		output_stream << encoded_data.get();
	}		}

			} // namespace base64
	/**
	* Get the length of base64-encoded block for given data byte length.
	*/
	inline std::size_t
	get_encoded_length(const std::size_t byte_length)
	{
	int encoded = std::ceil(byte_length * (4.0 / 3.0));
	// base64 uses padding to a multiple of 4
	if (encoded % 4 == 0)
	return encoded;
	return encoded + 4 - (encoded % 4);
	}

	} // namespace TNL		} // namespace TNL

src/TNL/zlib_compression.h

+13 −13

src/UnitTests/CMakeLists.txt

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -6,7 +6,7 @@ ADD_SUBDIRECTORY( Functions )
	ADD_SUBDIRECTORY( Meshes )		ADD_SUBDIRECTORY( Meshes )
	ADD_SUBDIRECTORY( Pointers )		ADD_SUBDIRECTORY( Pointers )

	set( CPP_TESTS AssertTest FileNameTest MathTest ObjectTest StringTest TimerTest TypeInfoTest )		set( CPP_TESTS AssertTest base64Test FileNameTest MathTest ObjectTest StringTest TimerTest TypeInfoTest )
	set( CUDA_TESTS AssertCudaTest )		set( CUDA_TESTS AssertCudaTest )
	if( BUILD_CUDA )		if( BUILD_CUDA )
	set( CUDA_TESTS ${CUDA_TESTS} AllocatorsTest FileTest )		set( CUDA_TESTS ${CUDA_TESTS} AllocatorsTest FileTest )

Original line number	Original line	Diff line number	Diff line
	@@ -187,7 +187,7 @@ protected:
	static std::size_t		static std::size_t
	readBlockSize( const char* block )		readBlockSize( const char* block )
	{		{
	std::pair<std::size_t, std::unique_ptr<char[]>> decoded_data = decode_block( block, get_encoded_length(sizeof(HeaderType)) );		std::pair<std::size_t, std::unique_ptr<std::uint8_t[]>> decoded_data = base64::decode( block, base64::get_encoded_length(sizeof(HeaderType)) );
	if( decoded_data.first != sizeof(HeaderType) )		if( decoded_data.first != sizeof(HeaderType) )
	throw MeshReaderError( "XMLVTK", "base64-decoding failed - mismatched data size in the binary header (read "		throw MeshReaderError( "XMLVTK", "base64-decoding failed - mismatched data size in the binary header (read "
	+ std::to_string(decoded_data.first) + " bytes, expected " + std::to_string(sizeof(HeaderType)) + " bytes)" );		+ std::to_string(decoded_data.first) + " bytes, expected " + std::to_string(sizeof(HeaderType)) + " bytes)" );
	@@ -205,8 +205,8 @@ protected:

	if( compressor == "" ) {		if( compressor == "" ) {
	const std::size_t blockSize = readBlockSize< HeaderType >( block );		const std::size_t blockSize = readBlockSize< HeaderType >( block );
	block += get_encoded_length(sizeof(HeaderType));		block += base64::get_encoded_length(sizeof(HeaderType));
	std::pair<std::size_t, std::unique_ptr<char[]>> decoded_data = decode_block( block, get_encoded_length(blockSize) );		std::pair<std::size_t, std::unique_ptr<std::uint8_t[]>> decoded_data = base64::decode( block, base64::get_encoded_length(blockSize) );
	std::vector<T> vector( decoded_data.first / sizeof(T) );		std::vector<T> vector( decoded_data.first / sizeof(T) );
	for( std::size_t i = 0; i < vector.size(); i++ )		for( std::size_t i = 0; i < vector.size(); i++ )
	vector[i] = reinterpret_cast<const T*>(decoded_data.second.get())[i];		vector[i] = reinterpret_cast<const T*>(decoded_data.second.get())[i];

Original line number	Original line	Diff line number	Diff line
	@@ -41,7 +41,7 @@ write_compressed_block(const T* data,

	// allocate a buffer for compressing data and do so		// allocate a buffer for compressing data and do so
	uLongf compressed_data_length = compressBound(data_size * sizeof(T));		uLongf compressed_data_length = compressBound(data_size * sizeof(T));
	std::unique_ptr<char[]> compressed_data{new char[compressed_data_length]};		std::unique_ptr<std::uint8_t[]> compressed_data{new std::uint8_t[compressed_data_length]};

	// compress the data		// compress the data
	const int status = compress2(		const int status = compress2(
	@@ -64,14 +64,14 @@ write_compressed_block(const T* data,

	// base64-encode the compression header		// base64-encode the compression header
	std::unique_ptr<char[]> encoded_header(		std::unique_ptr<char[]> encoded_header(
	encode_block(reinterpret_cast<const char*>(&compression_header[0]),		base64::encode(reinterpret_cast<const std::uint8_t*>(&compression_header[0]),
	4 * sizeof(HeaderType))		4 * sizeof(HeaderType))
	);		);
	output_stream << encoded_header.get();		output_stream << encoded_header.get();

	// base64-encode the compressed data		// base64-encode the compressed data
	std::unique_ptr<char[]> encoded_data(		std::unique_ptr<char[]> encoded_data(
	encode_block(compressed_data.get(), compressed_data_length)		base64::encode(compressed_data.get(), compressed_data_length)
	);		);
	output_stream << encoded_data.get();		output_stream << encoded_data.get();
	}		}
	@@ -82,7 +82,7 @@ write_compressed_block(const T* data,
	*/		*/
	template <typename T>		template <typename T>
	std::unique_ptr<T[]>		std::unique_ptr<T[]>
	decompress_data(const char* decoded_data, const std::size_t decoded_data_length, const std::size_t data_size)		decompress_data(const std::uint8_t* decoded_data, const std::size_t decoded_data_length, const std::size_t data_size)
	{		{
	// decompress the data		// decompress the data
	std::unique_ptr<T[]> data{new T[data_size]};		std::unique_ptr<T[]> data{new T[data_size]};
	@@ -116,8 +116,8 @@ std::pair<HeaderType, std::unique_ptr<T[]>>
	decompress_block(const char* data)		decompress_block(const char* data)
	{		{
	// decode the header		// decode the header
	const int encoded_header_length = get_encoded_length(4 * sizeof(HeaderType));		const int encoded_header_length = base64::get_encoded_length(4 * sizeof(HeaderType));
	std::pair<std::size_t, std::unique_ptr<char[]>> decoded_header = decode_block(data, encoded_header_length);		std::pair<std::size_t, std::unique_ptr<std::uint8_t[]>> decoded_header = base64::decode(data, encoded_header_length);
	const HeaderType* compression_header = reinterpret_cast<const HeaderType*>(decoded_header.second.get());		const HeaderType* compression_header = reinterpret_cast<const HeaderType*>(decoded_header.second.get());

	if (compression_header[0] != 1)		if (compression_header[0] != 1)