diff --git a/src/lepton/bitops.cc b/src/lepton/bitops.cc index 3032fb10..39ad8b0a 100644 --- a/src/lepton/bitops.cc +++ b/src/lepton/bitops.cc @@ -70,6 +70,7 @@ abitreader::~abitreader( void ) abitwriter::abitwriter( int size , int max_file_size) { + bits_written = 0; size_bound = max_file_size; if (size_bound) { size_bound += 8; // 64 bits of padding on the end diff --git a/src/lepton/bitops.hh b/src/lepton/bitops.hh index 2aa36e98..514784c7 100644 --- a/src/lepton/bitops.hh +++ b/src/lepton/bitops.hh @@ -73,7 +73,7 @@ public: int size_bound; public: void debug() const; - + uint32_t bits_written; abitwriter( int size, int size_bound); ~abitwriter( void ); @@ -117,7 +117,7 @@ public: void write( unsigned int val, int nbits ) { - + bits_written += nbits; int nbits2 = nbits; unsigned int val2 = val; dev_assert(nbits <= 64); diff --git a/src/lepton/lepton_codec.cc b/src/lepton/lepton_codec.cc index e18da758..bcb3da45 100644 --- a/src/lepton/lepton_codec.cc +++ b/src/lepton/lepton_codec.cc @@ -1,9 +1,83 @@ #include "lepton_codec.hh" #include "uncompressed_components.hh" #include "../vp8/decoder/decoder.hh" +#include +#include +#include "jpgcoder.hh" +std::mutex stats_mutex; +struct Stats{ + uint64_t time_us_lep; + uint64_t time_us_jpg; + uint32_t color; + uint16_t x; + uint16_t y; + uint16_t num_bits_lep; + uint16_t num_bits_jpg; + bool operator<(const Stats&other) const { + if (y != other.y) { + return y < other.y; + } + if (x != other.x) { + return x < other.x; + } + if (color != other.color) { + return color < other.color; + } + return time_us_jpg < other.time_us_jpg; + } +}; - - +std::vector cur_stats; +void add_lep_bits(uint32_t bits, uint16_t x, uint16_t y, int color) { + Stats s; + s.time_us_lep = TimingHarness::get_time_us(true); + s.time_us_jpg = 0; + s.color = (int)color; + s.x =x; + s.y = y; + s.num_bits_lep = bits; + s.num_bits_jpg = 0; + std::lock_guard l(stats_mutex); + cur_stats.push_back(s); + +} +void add_jpg_bits(uint32_t bits, uint16_t x, uint16_t y, int color) { + Stats s; + s.time_us_lep = 0; + s.time_us_jpg = TimingHarness::get_time_us(true); + s.color = (int)color; + s.x =x; + s.y = y; + s.num_bits_jpg = bits; + s.num_bits_lep = 0; + std::lock_guard l(stats_mutex); + cur_stats.push_back(s); +} +FILE * fs = fopen("/tmp/lepton_stats.csv", "w"); +void print_stats() { + std::sort(cur_stats.begin(), cur_stats.end()); + auto s = cur_stats.size(); + for (size_t i = 0;i < s; ++i) { + auto cur = cur_stats[i]; + if (cur.time_us_lep != 0) { + if ( i + 1 < s) { + auto next = cur_stats[i+1]; + if (cur.x == next.x && cur.y == next.y && cur.color == next.color) { + fprintf(fs, "%d,%d,%d, %d,%d, %ld,%ld\n", + cur.x, cur.y, cur.color, + cur.num_bits_lep, + next.num_bits_jpg, + cur.time_us_lep, + next.time_us_jpg); + } else { + fprintf(stderr, "MISMATCH %d != %d %d != %d %d != %d\n", cur.x, next.x, cur.y, next.y, cur.color, next.color); + } + } + } + } + fclose(fs); + fs = NULL; +} template void LeptonCodec::ThreadState::decode_row(Left & left_model, Middle& middle_model, @@ -14,10 +88,11 @@ void LeptonCodec::ThreadState::decode_row(Left & left_model, uint32_t block_width = image_data[(int)middle_model.COLOR]->block_width(); if (block_width > 0) { BlockContext context = context_.at((int)middle_model.COLOR); - parse_tokens(context, + uint32_t bits = parse_tokens(context, bool_decoder_, left_model, model_); //FIXME + add_lep_bits(bits, 0, curr_y, (int)middle_model.COLOR); int offset = image_data[middle_model.COLOR]->next(context_.at((int)middle_model.COLOR), true, curr_y); if (offset >= component_size_in_block) { return; @@ -25,23 +100,26 @@ void LeptonCodec::ThreadState::decode_row(Left & left_model, } for (unsigned int jpeg_x = 1; jpeg_x + 1 < block_width; jpeg_x++) { BlockContext context = context_.at((int)middle_model.COLOR); - parse_tokens(context, + uint32_t bits = parse_tokens(context, bool_decoder_, middle_model, model_); //FIXME int offset = image_data[middle_model.COLOR]->next(context_.at((int)middle_model.COLOR), true, curr_y); + add_lep_bits(bits, jpeg_x, curr_y, (int)middle_model.COLOR); if (offset >= component_size_in_block) { return; } } if (block_width > 1) { BlockContext context = context_.at((int)middle_model.COLOR); - parse_tokens(context, + + uint32_t bits = parse_tokens(context, bool_decoder_, right_model, model_); + add_lep_bits(bits, block_width - 1, curr_y, (int)middle_model.COLOR); image_data[middle_model.COLOR]->next(context_.at((int)middle_model.COLOR), false, curr_y); } } diff --git a/src/lepton/lepton_codec.hh b/src/lepton/lepton_codec.hh index 3dfd790a..c121b073 100644 --- a/src/lepton/lepton_codec.hh +++ b/src/lepton/lepton_codec.hh @@ -5,7 +5,8 @@ #include "bool_decoder.hh" #include "base_coders.hh" class UncompressedComponents; - +void print_stats(); +void add_jpg_bits(uint32_t bits, uint16_t x, uint16_t y, int color); class LeptonCodec { protected: struct ThreadState { diff --git a/src/lepton/recoder.cc b/src/lepton/recoder.cc index f3b45c2a..f1296bb7 100644 --- a/src/lepton/recoder.cc +++ b/src/lepton/recoder.cc @@ -343,10 +343,13 @@ bool recode_one_mcu_row(abitwriter *huffw, int mcu, lastdc[ cmp ] = dc; // encode block + uint32_t start_bit_pos = huffw->bits_written; int eob = encode_block_seq(huffw, &(hcodes[ 0 ][ cmpnfo[cmp].huffdc ]), &(hcodes[ 1 ][ cmpnfo[cmp].huffac ]), block.begin() ); + uint32_t end_bit_pos = huffw->bits_written; + add_jpg_bits(end_bit_pos - start_bit_pos, dpos%cmpnfo[cmp].bch, dpos/cmpnfo[cmp].bch, cmp); int old_mcu = mcu; // check for errors, proceed if no error encountered if ( eob < 0 ) sta = -1; @@ -842,6 +845,7 @@ bool recode_baseline_jpeg(bounded_iostream*str_out, str_out->write( grbgdata, grbs ); check_decompression_memory_bound_ok(); str_out->flush(); + print_stats(); TimingHarness::timing[0][TimingHarness::TS_JPEG_RECODE_FINISHED] = TimingHarness::get_time_us(); // errormessage if write error diff --git a/src/vp8/decoder/boolreader.cc b/src/vp8/decoder/boolreader.cc index a6745304..b52b8623 100644 --- a/src/vp8/decoder/boolreader.cc +++ b/src/vp8/decoder/boolreader.cc @@ -30,6 +30,7 @@ int vpx_reader_init(vpx_reader *r, r->value = 0; r->count = -8; r->range = 255; + r->bits_consumed=0; vpx_reader_fill(r); return vpx_read(r, 128, Billing::HEADER) != 0; // marker bit } diff --git a/src/vp8/decoder/boolreader.hh b/src/vp8/decoder/boolreader.hh index 0d0d3313..d20bb3ca 100644 --- a/src/vp8/decoder/boolreader.hh +++ b/src/vp8/decoder/boolreader.hh @@ -172,6 +172,7 @@ typedef struct { int count; BiRope buffer; PacketReader *reader; + uint32_t bits_consumed; // vpx_decrypt_cb decrypt_cb; // void *decrypt_state; } vpx_reader; @@ -360,6 +361,7 @@ inline bool vpx_reader_fill_and_read(vpx_reader *r, unsigned int split, Billing range <<= shift; value <<= shift; count -= shift; + r->bits_consumed+= shift; write_bit_bill(bill, true, shift); r->value = value; r->count = count; @@ -392,6 +394,7 @@ inline bool vpx_read(vpx_reader *r, int prob, Billing bill) { } //unsigned int shift = vpx_norm[range]; unsigned int shift = count_leading_zeros_uint8(range); + r->bits_consumed+= shift; range <<= shift; value <<= shift; count -= shift; diff --git a/src/vp8/decoder/decoder.cc b/src/vp8/decoder/decoder.cc index 804d2a7e..b466d923 100644 --- a/src/vp8/decoder/decoder.cc +++ b/src/vp8/decoder/decoder.cc @@ -165,10 +165,11 @@ void decode_edge(BlockContext mcontext, template -void parse_tokens(BlockContext context, +uint32_t parse_tokens(BlockContext context, BoolDecoder& decoder, ProbabilityTables & probability_tables, ProbabilityTablesBase &pt) { + uint32_t start_bits = decoder.bits_consumed(); context.here().bzero(); auto num_nonzeros_prob = probability_tables.nonzero_counts_7x7(pt, context.copy()); uint8_t num_nonzeros_7x7 = 0; @@ -315,15 +316,17 @@ void parse_tokens(BlockContext context, context.num_nonzeros_here->set_vertical(outp_sans_dc.begin(), ProbabilityTablesBase::quantization_table((int)color), context.here().dc()); + uint32_t end_bits = decoder.bits_consumed(); + return end_bits - start_bits; } #ifdef ALLOW_FOUR_COLORS -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); #endif -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); -template void parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); +template uint32_t parse_tokens(BlockContext, BoolDecoder&, ProbabilityTables&, ProbabilityTablesBase&); diff --git a/src/vp8/decoder/decoder.hh b/src/vp8/decoder/decoder.hh index 8c467d7d..55386dd4 100644 --- a/src/vp8/decoder/decoder.hh +++ b/src/vp8/decoder/decoder.hh @@ -2,7 +2,7 @@ #ifndef DECODER_HH #define DECODER_HH template -void parse_tokens(BlockContext context, +uint32_t parse_tokens(BlockContext context, BoolDecoder& data, ProbabilityTables & probability_tables, ProbabilityTablesBase&pt); diff --git a/src/vp8/decoder/vpx_bool_reader.hh b/src/vp8/decoder/vpx_bool_reader.hh index 14b1e67d..44ee19d0 100644 --- a/src/vp8/decoder/vpx_bool_reader.hh +++ b/src/vp8/decoder/vpx_bool_reader.hh @@ -35,6 +35,9 @@ public: #endif init(pr); } + uint32_t bits_consumed() const{ + return bit_reader.bits_consumed; + } #ifndef _WIN32 __attribute__((always_inline)) #endif