5 * Copyright 2012 - 2014
7 * Mathis Rosenhauer, Moritz Hanke, Joerg Behrens
8 * Deutsches Klimarechenzentrum GmbH
10 * 20146 Hamburg Germany
13 * Max-Planck-Institut fuer Meteorologie
18 * All rights reserved.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * 1. Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above
27 * copyright notice, this list of conditions and the following
28 * disclaimer in the documentation and/or other materials provided
29 * with the distribution.
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
34 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
35 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
36 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
37 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
38 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
39 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
40 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
42 * OF THE POSSIBILITY OF SUCH DAMAGE.
44 * @section DESCRIPTION
46 * Adaptive Entropy Decoder
47 * Based on CCSDS documents 121.0-B-2 and 120.0-G-3
64 #define BUFFERSPACE(strm) (strm->avail_in >= strm->state->in_blklen \
65 && strm->avail_out >= strm->state->out_blklen)
68 static void flush_##KIND(struct aec_stream *strm) \
70 uint32_t *flush_end, *bp, half_d; \
72 struct internal_state *state = strm->state; \
74 flush_end = state->rsip; \
76 if (state->flush_start == state->rsi_buffer \
77 && state->rsip > state->rsi_buffer) { \
78 state->last_out = *state->rsi_buffer; \
80 if (strm->flags & AEC_DATA_SIGNED) { \
81 m = UINT32_C(1) << (strm->bits_per_sample - 1); \
82 /* Reference samples have to be sign extended */ \
83 state->last_out = (state->last_out ^ m) - m; \
85 put_##KIND(strm, (uint32_t)state->last_out); \
86 state->flush_start++; \
89 data = state->last_out; \
91 if (state->xmin == 0) { \
92 uint32_t xmax, med, d; \
93 med = state->xmax / 2 + 1; \
96 for (bp = state->flush_start; bp < flush_end; bp++) { \
98 half_d = (d >> 1) + (d & 1); \
99 /*in this case: data >= med == data & med */ \
100 uint32_t mask = (data & med)?xmax:0; \
102 /*in this case: xmax - data == xmax ^ data */ \
103 if (half_d <= (mask ^ data)) { \
104 data += (d >> 1)^(~((d & 1) - 1)); \
108 put_##KIND(strm, (uint32_t)data); \
110 state->last_out = data; \
113 xmax = state->xmax; \
115 for (bp = state->flush_start; bp < flush_end; bp++) { \
117 half_d = ((uint32_t)d >> 1) + (d & 1); \
120 if (half_d <= xmax + data + 1) { \
121 data += ((uint32_t)d >> 1)^(~((d & 1) - 1)); \
123 data = d - xmax - 1; \
126 if (half_d <= xmax - data) { \
127 data += ((uint32_t)d >> 1)^(~((d & 1) - 1)); \
132 put_##KIND(strm, (uint32_t)data); \
134 state->last_out = data; \
137 for (bp = state->flush_start; bp < flush_end; bp++) \
138 put_##KIND(strm, *bp); \
140 state->flush_start = state->rsip; \
144 static inline void put_msb_32(struct aec_stream *strm, uint32_t data)
146 *strm->next_out++ = (unsigned char)(data >> 24);
147 *strm->next_out++ = (unsigned char)(data >> 16);
148 *strm->next_out++ = (unsigned char)(data >> 8);
149 *strm->next_out++ = (unsigned char)data;
152 static inline void put_msb_24(struct aec_stream *strm, uint32_t data)
154 *strm->next_out++ = (unsigned char)(data >> 16);
155 *strm->next_out++ = (unsigned char)(data >> 8);
156 *strm->next_out++ = (unsigned char)data;
159 static inline void put_msb_16(struct aec_stream *strm, uint32_t data)
161 *strm->next_out++ = (unsigned char)(data >> 8);
162 *strm->next_out++ = (unsigned char)data;
165 static inline void put_lsb_32(struct aec_stream *strm, uint32_t data)
167 *strm->next_out++ = (unsigned char)data;
168 *strm->next_out++ = (unsigned char)(data >> 8);
169 *strm->next_out++ = (unsigned char)(data >> 16);
170 *strm->next_out++ = (unsigned char)(data >> 24);
173 static inline void put_lsb_24(struct aec_stream *strm, uint32_t data)
175 *strm->next_out++ = (unsigned char)data;
176 *strm->next_out++ = (unsigned char)(data >> 8);
177 *strm->next_out++ = (unsigned char)(data >> 16);
180 static inline void put_lsb_16(struct aec_stream *strm, uint32_t data)
182 *strm->next_out++ = (unsigned char)data;
183 *strm->next_out++ = (unsigned char)(data >> 8);
186 static inline void put_8(struct aec_stream *strm, uint32_t data)
188 *strm->next_out++ = (unsigned char)data;
199 static inline void check_rsi_end(struct aec_stream *strm)
202 Flush output if end of RSI reached
204 struct internal_state *state = strm->state;
206 if (state->rsi_size == (size_t)(state->rsip - state->rsi_buffer)) {
207 state->flush_output(strm);
208 state->flush_start = state->rsi_buffer;
209 state->rsip = state->rsi_buffer;
213 static inline void put_sample(struct aec_stream *strm, uint32_t s)
215 struct internal_state *state = strm->state;
218 strm->avail_out -= state->bytes_per_sample;
222 static inline uint32_t direct_get(struct aec_stream *strm, int n)
225 Get n bit from input stream
227 No checking whatsoever. Read bits are dumped.
230 struct internal_state *state = strm->state;
235 b = (63 - state->bitp) >> 3;
237 state->acc = (state->acc << 48)
238 | ((uint64_t)strm->next_in[0] << 40)
239 | ((uint64_t)strm->next_in[1] << 32)
240 | ((uint64_t)strm->next_in[2] << 24)
241 | ((uint64_t)strm->next_in[3] << 16)
242 | ((uint64_t)strm->next_in[4] << 8)
243 | (uint64_t)strm->next_in[5];
245 state->acc = (state->acc << 56)
246 | ((uint64_t)strm->next_in[0] << 48)
247 | ((uint64_t)strm->next_in[1] << 40)
248 | ((uint64_t)strm->next_in[2] << 32)
249 | ((uint64_t)strm->next_in[3] << 24)
250 | ((uint64_t)strm->next_in[4] << 16)
251 | ((uint64_t)strm->next_in[5] << 8)
252 | (uint64_t)strm->next_in[6];
254 state->acc = (state->acc << 40)
255 | ((uint64_t)strm->next_in[0] << 32)
256 | ((uint64_t)strm->next_in[1] << 24)
257 | ((uint64_t)strm->next_in[2] << 16)
258 | ((uint64_t)strm->next_in[3] << 8)
259 | (uint64_t)strm->next_in[4];
261 state->acc = (state->acc << 32)
262 | ((uint64_t)strm->next_in[0] << 24)
263 | ((uint64_t)strm->next_in[1] << 16)
264 | ((uint64_t)strm->next_in[2] << 8)
265 | (uint64_t)strm->next_in[3];
267 state->acc = (state->acc << 24)
268 | ((uint64_t)strm->next_in[0] << 16)
269 | ((uint64_t)strm->next_in[1] << 8)
270 | (uint64_t)strm->next_in[2];
272 state->acc = (state->acc << 16)
273 | ((uint64_t)strm->next_in[0] << 8)
274 | (uint64_t)strm->next_in[1];
276 state->acc = (state->acc << 8)
277 | (uint64_t)strm->next_in[0];
281 state->bitp += b << 3;
285 return (state->acc >> state->bitp) & (UINT64_MAX >> (64 - n));
288 static inline uint32_t direct_get_fs(struct aec_stream *strm)
291 Interpret a Fundamental Sequence from the input buffer.
293 Essentially counts the number of 0 bits until a 1 is
303 struct internal_state *state = strm->state;
306 state->acc &= UINT64_MAX >> (64 - state->bitp);
310 while (state->acc == 0) {
311 state->acc = (state->acc << 56)
312 | ((uint64_t)strm->next_in[0] << 48)
313 | ((uint64_t)strm->next_in[1] << 40)
314 | ((uint64_t)strm->next_in[2] << 32)
315 | ((uint64_t)strm->next_in[3] << 24)
316 | ((uint64_t)strm->next_in[4] << 16)
317 | ((uint64_t)strm->next_in[5] << 8)
318 | (uint64_t)strm->next_in[6];
325 #if HAVE_DECL___BUILTIN_CLZLL
326 i = 63 - __builtin_clzll(state->acc);
328 _BitScanReverse64(&i, state->acc);
331 while ((state->acc & (UINT64_C(1) << i)) == 0)
334 fs += state->bitp - i - 1;
339 static inline uint32_t bits_ask(struct aec_stream *strm, int n)
341 while (strm->state->bitp < n) {
342 if (strm->avail_in == 0)
345 strm->state->acc <<= 8;
346 strm->state->acc |= *strm->next_in++;
347 strm->state->bitp += 8;
352 static inline uint32_t bits_get(struct aec_stream *strm, int n)
354 return (strm->state->acc >> (strm->state->bitp - n))
355 & (UINT64_MAX >> (64 - n));
358 static inline void bits_drop(struct aec_stream *strm, int n)
360 strm->state->bitp -= n;
363 static inline uint32_t fs_ask(struct aec_stream *strm)
365 if (bits_ask(strm, 1) == 0)
367 while ((strm->state->acc & (UINT64_C(1) << (strm->state->bitp - 1))) == 0) {
368 if (strm->state->bitp == 1) {
369 if (strm->avail_in == 0)
372 strm->state->acc <<= 8;
373 strm->state->acc |= *strm->next_in++;
374 strm->state->bitp += 8;
382 static inline void fs_drop(struct aec_stream *strm)
388 static inline uint32_t copysample(struct aec_stream *strm)
390 if (bits_ask(strm, strm->bits_per_sample) == 0
391 || strm->avail_out == 0)
394 put_sample(strm, bits_get(strm, strm->bits_per_sample));
395 bits_drop(strm, strm->bits_per_sample);
399 static int m_id(struct aec_stream *strm)
401 struct internal_state *state = strm->state;
403 if (state->rsip == state->rsi_buffer) {
404 if(strm->flags & AEC_PAD_RSI)
405 state->bitp -= state->bitp % 8;
411 if (bits_ask(strm, state->id_len) == 0)
413 state->id = bits_get(strm, state->id_len);
414 bits_drop(strm, state->id_len);
415 state->mode = state->id_table[state->id];
420 static int m_split_output(struct aec_stream *strm)
422 struct internal_state *state = strm->state;
423 int k = state->id - 1;
426 if (bits_ask(strm, k) == 0 || strm->avail_out == 0)
429 *state->rsip++ += bits_get(strm, k);
432 strm->avail_out -= state->bytes_per_sample;
434 } while(++state->i < state->n);
441 static int m_split_fs(struct aec_stream *strm)
443 struct internal_state *state = strm->state;
444 int k = state->id - 1;
447 if (fs_ask(strm) == 0)
449 state->rsip[state->i] = state->fs << k;
451 } while(++state->i < state->n);
454 state->mode = m_split_output;
459 static int m_split(struct aec_stream *strm)
462 struct internal_state *state = strm->state;
464 if (BUFFERSPACE(strm)) {
468 *state->rsip++ = direct_get(strm, strm->bits_per_sample);
470 for (i = 0; i < strm->block_size - state->ref; i++)
471 state->rsip[i] = direct_get_fs(strm) << k;
474 for (i = state->ref; i < strm->block_size; i++)
475 *state->rsip++ += direct_get(strm, k);
477 state->rsip += strm->block_size - state->ref;
480 strm->avail_out -= state->out_blklen;
488 if (copysample(strm) == 0)
490 state->n = strm->block_size - 1;
492 state->n = strm->block_size;
496 state->mode = m_split_fs;
500 static int m_zero_output(struct aec_stream *strm)
502 struct internal_state *state = strm->state;
505 if (strm->avail_out == 0)
514 static int m_zero_block(struct aec_stream *strm)
516 uint32_t i, zero_blocks, b, zero_bytes;
517 struct internal_state *state = strm->state;
519 if (fs_ask(strm) == 0)
521 zero_blocks = state->fs + 1;
524 if (zero_blocks == ROS) {
525 b = (int)(state->rsip - state->rsi_buffer) / strm->block_size;
526 zero_blocks = MIN(strm->rsi - b, 64 - (b % 64));
527 } else if (zero_blocks > ROS) {
532 i = zero_blocks * strm->block_size - 1;
534 i = zero_blocks * strm->block_size;
536 zero_bytes = i * state->bytes_per_sample;
538 if (strm->avail_out >= zero_bytes) {
539 if (state->rsi_size - (state->rsip - state->rsi_buffer) < i)
542 memset(state->rsip, 0, i * sizeof(uint32_t));
544 strm->avail_out -= zero_bytes;
552 state->mode = m_zero_output;
556 static int m_se_decode(struct aec_stream *strm)
559 struct internal_state *state = strm->state;
561 while(state->i < strm->block_size) {
562 if (fs_ask(strm) == 0)
565 d1 = m - state->se_table[2 * m + 1];
567 if ((state->i & 1) == 0) {
568 if (strm->avail_out == 0)
570 put_sample(strm, state->se_table[2 * m] - d1);
574 if (strm->avail_out == 0)
576 put_sample(strm, d1);
585 static int m_se(struct aec_stream *strm)
589 struct internal_state *state = strm->state;
591 if (BUFFERSPACE(strm)) {
594 while (i < strm->block_size) {
595 m = direct_get_fs(strm);
596 d1 = m - state->se_table[2 * m + 1];
599 put_sample(strm, state->se_table[2 * m] - d1);
602 put_sample(strm, d1);
609 state->mode = m_se_decode;
610 state->i = state->ref;
614 static int m_low_entropy_ref(struct aec_stream *strm)
616 struct internal_state *state = strm->state;
618 if (state->ref && copysample(strm) == 0)
626 state->mode = m_zero_block;
630 static int m_low_entropy(struct aec_stream *strm)
632 struct internal_state *state = strm->state;
634 if (bits_ask(strm, 1) == 0)
636 state->id = bits_get(strm, 1);
638 state->mode = m_low_entropy_ref;
642 static int m_uncomp_copy(struct aec_stream *strm)
644 struct internal_state *state = strm->state;
647 if (copysample(strm) == 0)
655 static int m_uncomp(struct aec_stream *strm)
658 struct internal_state *state = strm->state;
660 if (BUFFERSPACE(strm)) {
661 for (i = 0; i < strm->block_size; i++)
662 *state->rsip++ = direct_get(strm, strm->bits_per_sample);
663 strm->avail_out -= state->out_blklen;
670 state->i = strm->block_size;
671 state->mode = m_uncomp_copy;
675 static void create_se_table(int *table)
680 for (i = 0; i < 13; i++) {
682 for (j = 0; j <= i; j++) {
684 table[2 * k + 1] = ms;
690 int aec_decode_init(struct aec_stream *strm)
693 struct internal_state *state;
695 if (strm->bits_per_sample > 32 || strm->bits_per_sample == 0)
696 return AEC_CONF_ERROR;
698 state = malloc(sizeof(struct internal_state));
700 return AEC_MEM_ERROR;
701 memset(state, 0, sizeof(struct internal_state));
703 create_se_table(state->se_table);
707 if (strm->bits_per_sample > 16) {
710 if (strm->bits_per_sample <= 24 && strm->flags & AEC_DATA_3BYTE) {
711 state->bytes_per_sample = 3;
712 if (strm->flags & AEC_DATA_MSB)
713 state->flush_output = flush_msb_24;
715 state->flush_output = flush_lsb_24;
717 state->bytes_per_sample = 4;
718 if (strm->flags & AEC_DATA_MSB)
719 state->flush_output = flush_msb_32;
721 state->flush_output = flush_lsb_32;
723 state->out_blklen = strm->block_size
724 * state->bytes_per_sample;
726 else if (strm->bits_per_sample > 8) {
727 state->bytes_per_sample = 2;
729 state->out_blklen = strm->block_size * 2;
730 if (strm->flags & AEC_DATA_MSB)
731 state->flush_output = flush_msb_16;
733 state->flush_output = flush_lsb_16;
735 if (strm->flags & AEC_RESTRICTED) {
736 if (strm->bits_per_sample <= 4) {
737 if (strm->bits_per_sample <= 2)
742 return AEC_CONF_ERROR;
748 state->bytes_per_sample = 1;
749 state->out_blklen = strm->block_size;
750 state->flush_output = flush_8;
753 if (strm->flags & AEC_DATA_SIGNED) {
754 state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1);
755 state->xmin = ~state->xmax;
758 state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample);
761 state->in_blklen = (strm->block_size * strm->bits_per_sample
762 + state->id_len) / 8 + 9;
764 modi = 1UL << state->id_len;
765 state->id_table = malloc(modi * sizeof(int (*)(struct aec_stream *)));
766 if (state->id_table == NULL)
767 return AEC_MEM_ERROR;
769 state->id_table[0] = m_low_entropy;
770 for (i = 1; i < modi - 1; i++) {
771 state->id_table[i] = m_split;
773 state->id_table[modi - 1] = m_uncomp;
775 state->rsi_size = strm->rsi * strm->block_size;
776 state->rsi_buffer = malloc(state->rsi_size * sizeof(uint32_t));
777 if (state->rsi_buffer == NULL)
778 return AEC_MEM_ERROR;
784 state->rsip = state->rsi_buffer;
785 state->flush_start = state->rsi_buffer;
788 state->pp = strm->flags & AEC_DATA_PREPROCESS;
793 int aec_decode(struct aec_stream *strm, int flush)
796 Finite-state machine implementation of the adaptive entropy
799 Can work with one byte input und one sample output buffers. If
800 enough buffer space is available, then faster implementations
801 of the states are called. Inspired by zlib.
804 struct internal_state *state = strm->state;
807 strm->total_in += strm->avail_in;
808 strm->total_out += strm->avail_out;
811 status = state->mode(strm);
812 } while (status == M_CONTINUE);
814 if (status == M_ERROR)
815 return AEC_DATA_ERROR;
817 state->flush_output(strm);
819 strm->total_in -= strm->avail_in;
820 strm->total_out -= strm->avail_out;
825 int aec_decode_end(struct aec_stream *strm)
827 struct internal_state *state = strm->state;
829 free(state->id_table);
830 free(state->rsi_buffer);
835 int aec_buffer_decode(struct aec_stream *strm)
839 status = aec_decode_init(strm);
840 if (status != AEC_OK)
843 status = aec_decode(strm, AEC_FLUSH);
844 aec_decode_end(strm);