Implemented binary and zlib-compressed data formats for VTUWriter (32d1c833) · Commits · TNL / tnl-dev

src/TNL/Meshes/Writers/VTKWriter.hpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -12,9 +12,9 @@

		#include <limits>

		#include <TNL/Endianness.h>
		#include <TNL/Meshes/Writers/VTKWriter.h>
		#include <TNL/Meshes/Writers/VerticesPerEntity.h>
		#include <TNL/Endianness.h>

		namespace TNL {
		namespace Meshes {

src/TNL/Meshes/Writers/VTUWriter.hpp

+28 −10

Original line number	Diff line number	Diff line
		@@ -14,9 +14,13 @@

		#include <limits>

		#include <TNL/Endianness.h>
		#include <TNL/Meshes/Writers/VTUWriter.h>
		#include <TNL/Meshes/Writers/VerticesPerEntity.h>
		#include <TNL/Endianness.h>
		#include <TNL/base64.h>
		#ifdef HAVE_ZLIB
		#include <TNL/zlib_compression.h>
		#endif

		namespace TNL {
		namespace Meshes {
		@@ -472,7 +476,9 @@ VTUWriter< Mesh >::writeDataArray( const Array& array,
		str << " NumberOfComponents=\"" << numberOfComponents << "\"";
		str << " format=\"" << ((format == VTK::FileFormat::ascii) ? "ascii" : "binary") << "\">\n";

		if( format == VTK::FileFormat::ascii ) {
		switch( format )
		{
		case VTK::FileFormat::ascii:
		str.precision( std::numeric_limits< typename Array::ValueType >::digits10 );
		for( IndexType i = 0; i < array.getSize(); i++ )
		// If Array::ValueType is uint8_t, it might be a typedef for unsigned char, which
		@@ -480,8 +486,20 @@ VTUWriter< Mesh >::writeDataArray( const Array& array,
		// with unary operator+, see https://stackoverflow.com/a/28414758
		str << +array[i] << " ";
		str << "\n";
		break;
		case VTK::FileFormat::binary:
		write_encoded_block< HeaderType >( array.getData(), array.getSize(), str );
		str << "\n";
		break;
		case VTK::FileFormat::zlib_compressed:
		#ifdef HAVE_ZLIB
		write_compressed_block< HeaderType >( array.getData(), array.getSize(), str );
		str << "\n";
		break;
		#else
		throw std::runtime_error("The ZLIB compression algorithm is not available in this build. Please recompile the program with -DHAVE_ZLIB.");
		#endif
		}
		// TODO: binary format, compression

		// write DataArray footer
		str << "</DataArray>\n";

src/TNL/base64.h

0 → 100644

+316 −0

Original line number	Diff line number	Diff line
		/***************************************************************************
		base64.h - description
		-------------------
		begin : Mar 20, 2020
		copyright : (C) 2020 by Tomas Oberhuber et al.
		email : tomas.oberhuber@fjfi.cvut.cz
		***************************************************************************/

		/* See Copyright Notice in tnl/Copyright */

		#pragma once

		#include <cstddef>
		#include <memory>
		#include <utility>
		#include <cmath> // std::ceil

		namespace TNL {

		// The functions in the base64 namespace are taken from the libb64 project, see
		// http://sourceforge.net/projects/libb64
		//
		// libb64 has been placed in the public domain
		namespace base64 {

		// encoding

		typedef enum
		{
		step_A,
		step_B,
		step_C
		} base64_encodestep;

		typedef struct
		{
		base64_encodestep step;
		char result;
		} base64_encodestate;

		inline void
		base64_init_encodestate(base64_encodestate *state_in)
		{
		state_in->step = step_A;
		state_in->result = 0;
		}

		inline char
		base64_encode_value(char value_in)
		{
		static const char *encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
		if (value_in > 63)
		return '=';
		return encoding[(int)value_in];
		}

		inline std::ptrdiff_t
		base64_encode_block(const char * plaintext_in,
		std::size_t length_in,
		char * code_out,
		base64_encodestate *state_in)
		{
		const char * plainchar = plaintext_in;
		const char *const plaintextend = plaintext_in + length_in;
		char * codechar = code_out;
		char result;

		result = state_in->result;

		switch (state_in->step)
		{
		while (true)
		{
		case step_A:
		{
		if (plainchar == plaintextend)
		{
		state_in->result = result;
		state_in->step = step_A;
		return codechar - code_out;
		}
		const char fragment = *plainchar++;
		result = (fragment & 0x0fc) >> 2;
		*codechar++ = base64_encode_value(result);
		result = (fragment & 0x003) << 4;
		// intended fallthrough
		}
		case step_B:
		{
		if (plainchar == plaintextend)
		{
		state_in->result = result;
		state_in->step = step_B;
		return codechar - code_out;
		}
		const char fragment = *plainchar++;
		result \|= (fragment & 0x0f0) >> 4;
		*codechar++ = base64_encode_value(result);
		result = (fragment & 0x00f) << 2;
		// intended fallthrough
		}
		case step_C:
		{
		if (plainchar == plaintextend)
		{
		state_in->result = result;
		state_in->step = step_C;
		return codechar - code_out;
		}
		const char fragment = *plainchar++;
		result \|= (fragment & 0x0c0) >> 6;
		*codechar++ = base64_encode_value(result);
		result = (fragment & 0x03f) >> 0;
		*codechar++ = base64_encode_value(result);
		}
		}
		}
		/* control should not reach here */
		return codechar - code_out;
		}

		inline std::ptrdiff_t
		base64_encode_blockend(char code_out, base64_encodestate state_in)
		{
		char *codechar = code_out;

		switch (state_in->step)
		{
		case step_B:
		*codechar++ = base64_encode_value(state_in->result);
		*codechar++ = '=';
		*codechar++ = '=';
		break;
		case step_C:
		*codechar++ = base64_encode_value(state_in->result);
		*codechar++ = '=';
		break;
		case step_A:
		break;
		}
		*codechar++ = '\0';

		return codechar - code_out;
		}


		// decoding

		typedef enum
		{
		step_a, step_b, step_c, step_d
		} base64_decodestep;

		typedef struct
		{
		base64_decodestep step;
		char plainchar;
		} base64_decodestate;

		inline void
		base64_init_decodestate(base64_decodestate* state_in)
		{
		state_in->step = step_a;
		state_in->plainchar = 0;
		}

		inline int
		base64_decode_value(char value_in)
		{
		static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51};
		static const char decoding_size = sizeof(decoding);
		value_in -= 43;
		if (value_in < 0 \|\| value_in >= decoding_size) return -1;
		return decoding[(int)value_in];
		}

		inline std::ptrdiff_t
		base64_decode_block(const char* code_in, const std::size_t length_in, char* plaintext_out, base64_decodestate* state_in)
		{
		const char* codechar = code_in;
		char* plainchar = plaintext_out;
		char fragment;

		*plainchar = state_in->plainchar;

		switch (state_in->step)
		{
		while (1)
		{
		case step_a:
		do {
		if (codechar == code_in+length_in)
		{
		state_in->step = step_a;
		state_in->plainchar = *plainchar;
		return plainchar - plaintext_out;
		}
		fragment = (char)base64_decode_value(*codechar++);
		} while (fragment < 0);
		*plainchar = (fragment & 0x03f) << 2;
		case step_b:
		do {
		if (codechar == code_in+length_in)
		{
		state_in->step = step_b;
		state_in->plainchar = *plainchar;
		return plainchar - plaintext_out;
		}
		fragment = (char)base64_decode_value(*codechar++);
		} while (fragment < 0);
		*plainchar++ \|= (fragment & 0x030) >> 4;
		*plainchar = (fragment & 0x00f) << 4;
		case step_c:
		do {
		if (codechar == code_in+length_in)
		{
		state_in->step = step_c;
		state_in->plainchar = *plainchar;
		return plainchar - plaintext_out;
		}
		fragment = (char)base64_decode_value(*codechar++);
		} while (fragment < 0);
		*plainchar++ \|= (fragment & 0x03c) >> 2;
		*plainchar = (fragment & 0x003) << 6;
		case step_d:
		do {
		if (codechar == code_in+length_in)
		{
		state_in->step = step_d;
		state_in->plainchar = *plainchar;
		return plainchar - plaintext_out;
		}
		fragment = (char)base64_decode_value(*codechar++);
		} while (fragment < 0);
		*plainchar++ \|= (fragment & 0x03f);
		}
		}
		/* control should not reach here */
		return plainchar - plaintext_out;
		}

		} // namespace base64


		/**
		* Do a base64 encoding of the given data.
		*
		* The function returns a unique_ptr to the encoded data.
		*/
		inline std::unique_ptr<char[]>
		encode_block(const char* data, const std::size_t data_size)
		{
		base64::base64_encodestate state;
		base64::base64_init_encodestate(&state);

		std::unique_ptr<char[]> encoded_data{new char[2 * data_size + 1]};

		const std::size_t encoded_length_data = base64::base64_encode_block(data, data_size, encoded_data.get(), &state);
		base64::base64_encode_blockend(encoded_data.get() + encoded_length_data, &state);

		return encoded_data;
		}


		/**
		* Do a base64 decoding of the given data.
		*
		* The function returns a pair of the decoded data length and a unique_ptr to
		* the decoded data.
		*/
		inline std::pair<std::size_t, std::unique_ptr<char[]>>
		decode_block(const char* data, const std::size_t data_size)
		{
		base64::base64_decodestate state;
		base64::base64_init_decodestate(&state);

		std::unique_ptr<char[]> decoded_data{new char[data_size + 1]};

		const std::size_t decoded_length_data = base64::base64_decode_block(data, data_size, decoded_data.get(), &state);
		decoded_data[decoded_length_data] = '\0';

		return {decoded_length_data, std::move(decoded_data)};
		}


		/**
		* Write a base64-encoded block of data into the given stream.
		*
		* The encoded data is prepended with a short header, which is the base64-encoded
		* byte length of the data. The type of the byte length value is `HeaderType`.
		*/
		template <typename HeaderType = std::uint64_t, typename T>
		void write_encoded_block(const T* data, const std::size_t data_length, std::ostream& output_stream)
		{
		const HeaderType size = data_length * sizeof(T);
		std::unique_ptr<char[]> encoded_size = encode_block(reinterpret_cast<const char*>(&size), sizeof(HeaderType));
		output_stream << encoded_size.get();
		std::unique_ptr<char[]> encoded_data = encode_block(reinterpret_cast<const char*>(data), size);
		output_stream << encoded_data.get();
		}


		/**
		* Get the length of base64-encoded block for given data byte length.
		*/
		inline std::size_t
		get_encoded_length(const std::size_t byte_length)
		{
		int encoded = std::ceil(byte_length * (4.0 / 3.0));
		// base64 uses padding to a multiple of 4
		if (encoded % 4 == 0)
		return encoded;
		return encoded + 4 - (encoded % 4);
		}

		} // namespace TNL

src/TNL/zlib_compression.h

0 → 100644

+185 −0

Original line number	Diff line number	Diff line
		/***************************************************************************
		zlib_compression.h - description
		-------------------
		begin : Mar 20, 2020
		copyright : (C) 2020 by Tomas Oberhuber et al.
		email : tomas.oberhuber@fjfi.cvut.cz
		***************************************************************************/

		/* See Copyright Notice in tnl/Copyright */

		#pragma once

		#include <memory>
		#include <cstdint> // std::uint64_t

		#include <zlib.h>

		#include <TNL/base64.h>

		namespace TNL {

		/**
		* Use zlib to compress the data, then encode it with base64 and write the
		* result to the given stream.
		*
		* Options for compression_level:
		* - Z_NO_COMPRESSION
		* - Z_BEST_SPEED
		* - Z_BEST_COMPRESSION
		* - Z_DEFAULT_COMPRESSION
		*/
		template <typename HeaderType = std::uint64_t, typename T>
		void
		write_compressed_block(const T* data,
		const std::size_t data_size,
		std::ostream& output_stream,
		const int compression_level = Z_DEFAULT_COMPRESSION)
		{
		if (data_size == 0)
		return;

		// allocate a buffer for compressing data and do so
		uLongf compressed_data_length = compressBound(data_size * sizeof(T));
		std::unique_ptr<char[]> compressed_data{new char[compressed_data_length]};

		// compress the data
		const int status = compress2(
		reinterpret_cast<Bytef*>(compressed_data.get()),
		&compressed_data_length,
		reinterpret_cast<const Bytef*>(data),
		data_size * sizeof(T),
		compression_level
		);
		if (status != Z_OK)
		throw std::runtime_error("zlib compression failed");

		// create compression header
		const HeaderType compression_header[4] = {
		(HeaderType) 1, // number of blocks
		(HeaderType) (data_size * sizeof(T)), // size of block
		(HeaderType) (data_size * sizeof(T)), // size of last block
		(HeaderType) compressed_data_length // list of compressed sizes of blocks
		};

		// base64-encode the compression header
		std::unique_ptr<char[]> encoded_header(
		encode_block(reinterpret_cast<const char*>(&compression_header[0]),
		4 * sizeof(HeaderType))
		);
		output_stream << encoded_header.get();

		// base64-encode the compressed data
		std::unique_ptr<char[]> encoded_data(
		encode_block(compressed_data.get(), compressed_data_length)
		);
		output_stream << encoded_data.get();
		}

		/**
		* Decompress data in given byte array and return an array of elements of
		* type T and length data_size.
		*/
		template <typename T>
		std::unique_ptr<T[]>
		decompress_data(const char* decoded_data, const std::size_t decoded_data_length, const std::size_t data_size)
		{
		// decompress the data
		std::unique_ptr<T[]> data{new T[data_size]};
		uLongf uncompressed_length_data = data_size * sizeof(T);
		const int status = uncompress(
		reinterpret_cast<Bytef*>(data.get()),
		&uncompressed_length_data,
		reinterpret_cast<const Bytef*>(decoded_data),
		decoded_data_length
		);
		if (status != Z_OK)
		throw std::runtime_error("zlib decompression failed");

		if (uncompressed_length_data != data_size * sizeof(T))
		throw std::length_error("uncompressed data length does not match the size stored in the compression header: "
		+ std::to_string(uncompressed_length_data) + " vs "
		+ std::to_string(data_size));

		return data;
		}

		/**
		* Decode and decompress data from a stream. The data must be in the format
		* as produced by the write_compressed_block function.
		*
		* The function returns a pair of the resulting data size and a unique_ptr to
		* the data.
		*/
		template <typename HeaderType = std::uint64_t, typename T>
		std::pair<HeaderType, std::unique_ptr<T[]>>
		decompress_block(const char* data)
		{
		// decode the header
		const int encoded_header_length = get_encoded_length(4 * sizeof(HeaderType));
		std::pair<std::size_t, std::unique_ptr<char[]>> decoded_header = decode_block(data, encoded_header_length);
		const HeaderType* compression_header = reinterpret_cast<const HeaderType*>(decoded_header.second.get());

		if (compression_header[0] != 1)
		throw std::length_error("unexpected number of compressed blocks: "
		+ std::to_string(compression_header[0]));
		if (compression_header[1] != compression_header[2])
		throw std::logic_error("inconsistent block sizes in the compression header: "
		+ std::to_string(compression_header[1]) + " vs "
		+ std::to_string(compression_header[2]));
		const HeaderType data_size = compression_header[1] / sizeof(T);
		const HeaderType compressed_data_length = get_encoded_length(compression_header[3]);

		// decode the data
		std::pair<std::size_t, std::unique_ptr<char[]>> decoded_data = decode_block(data + encoded_header_length, compressed_data_length);

		// decompress the data and return
		return {data_size, decompress_data<T>(decoded_data.second.get(), decoded_data.first, data_size)};
		}

		/**
		* Decode and decompress data from a stream. The data must be in the format
		* as produced by the write_compressed_block function.
		*
		* The function returns a pair of the resulting data size and a unique_ptr to
		* the data.
		*/
		template <typename HeaderType = std::uint64_t, typename T>
		std::pair<HeaderType, std::unique_ptr<T[]>>
		decompress_block(std::istream& input_stream)
		{
		// read the header
		const int encoded_header_length = get_encoded_length(4 * sizeof(HeaderType));
		std::unique_ptr<char[]> encoded_header{new char[encoded_header_length]};
		input_stream.read(encoded_header.get(), encoded_header_length);
		if (!input_stream.good())
		throw std::length_error("input is not long enough to contain a compression header");

		// decode the header
		std::pair<std::size_t, std::unique_ptr<char[]>> decoded_header = decode_block(encoded_header.get(), encoded_header_length);
		const HeaderType* compression_header = reinterpret_cast<const HeaderType*>(decoded_header.second.get());

		if (compression_header[0] != 1)
		throw std::length_error("unexpected number of compressed blocks: "
		+ std::to_string(compression_header[0]));
		if (compression_header[1] != compression_header[2])
		throw std::logic_error("inconsistent block sizes in the compression header: "
		+ std::to_string(compression_header[1]) + " vs "
		+ std::to_string(compression_header[2]));
		const HeaderType data_size = compression_header[1] / sizeof(T);
		const HeaderType compressed_data_length = get_encoded_length(compression_header[3]);

		// read the compressed data
		std::unique_ptr<char[]> encoded_data{new char[compressed_data_length]};
		input_stream.read(encoded_data.get(), compressed_data_length);
		if (!input_stream.good())
		throw std::length_error("failed to read the compressed data");

		// decode the data
		std::pair<std::size_t, std::unique_ptr<char[]>> decoded_data = decode_block(encoded_data.get(), compressed_data_length);

		// decompress the data and return
		return {data_size, decompress_data<T>(decoded_data.second.get(), decoded_data.first, data_size)};
		}

		} // namespace TNL