llimagej2ckdu.cpp

/** 
 * @file llimagej2ckdu.cpp
 * @brief This is an implementation of JPEG2000 encode/decode using Kakadu
 *
 * $LicenseInfo:firstyear=2010&license=viewerlgpl$
 * Second Life Viewer Source Code
 * Copyright (C) 2010, Linden Research, Inc.
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation;
 * version 2.1 of the License only.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 * 
 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
 * $/LicenseInfo$
 */

#include "linden_common.h"

#include "llimagej2ckdu.h"

#include "lltimer.h"
#include "llpointer.h"
#include "llmath.h"
#include "llkdumem.h"

#define kdu_xxxx "kdu_block_coding.h"
#include "include_kdu_xxxx.h"

// Avoid ubiquitous necessity of kdu_core:: qualification
using namespace kdu_core;

#include "llexception.h"
#include <boost/exception/diagnostic_information.hpp>
#include <sstream>
#include <iomanip>

// Turns out this must NOT be in the anonymous namespace!
namespace kdu_core
{
// stream kdu_dims to std::ostream
inline
std::ostream& operator<<(std::ostream& out, const kdu_dims& dims)
{
	return out << "(" << dims.pos.x << "," << dims.pos.y << "),"
				  "[" << dims.size.x << "x" << dims.size.y << "]";
}
} // namespace kdu_core

// operator<<(std::ostream&, const kdu_dims&) must precede #include "stringize.h"
#include "stringize.h"

namespace {
// Failure to load an image shouldn't crash the whole viewer.
struct KDUError: public LLContinueError
{
    KDUError(const std::string& msg): LLContinueError(msg) {}
};

// KDU defines int error codes as hex values, so we should log them in hex
// so we can grep KDU headers for the hex. However those hex values
// generally "happen" to encode big-endian multibyte character sequences,
// e.g. KDU_ERROR_EXCEPTION is 0x6b647545: 'kduE'
// But beware because KDU_NULL_EXCEPTION is simply 0 -- which doesn't
// preclude somebody from throwing it.
std::string report_kdu_exception(kdu_exception mb)
{
    std::ostringstream out;
    // always report mb in hex
    out << "kdu_exception " << std::hex << mb;

    // Also display as many chars as are encoded in the kdu_exception
    // value. Make a char array; reserve 1 extra byte for nul terminator.
    char bytes[sizeof(kdu_exception) + 1];
    // Back up through 'bytes'
    char *bptr = bytes + sizeof(bytes);
    *(--bptr) = '\0';
    while (mb)
    {
        // store low-order byte of mb in next-left char
        *(--bptr) = char(mb & 0xFF);
        // then shift mb right by one byte
        mb >>= 8;
    }
    // did that produce any characters?
    if (*bptr)
    {
        out << " (" << bptr << ')';
    }

    return out.str();
}
} // anonymous namespace


class kdc_flow_control {
	
public:
	kdc_flow_control(kdu_supp::kdu_image_in_base *img_in, kdu_codestream codestream);
	~kdc_flow_control();
	bool advance_components();
	void process_components();
	
private:
	
	struct kdc_component_flow_control {
	public:
		kdu_supp::kdu_image_in_base *reader;
		int vert_subsampling;
		int ratio_counter;  /*  Initialized to 0, decremented by `count_delta';
                                when < 0, a new line must be processed, after
                                which it is incremented by `vert_subsampling'.  */
		int initial_lines;
		int remaining_lines;
		kdu_line_buf *line;
	};
	
	kdu_codestream codestream;
	kdu_dims valid_tile_indices;
	kdu_coords tile_idx;
	kdu_tile tile;
	int num_components;
	kdc_component_flow_control *components;
	int count_delta; // Holds the minimum of the `vert_subsampling' fields
	kdu_multi_analysis engine;
	kdu_long max_buffer_memory;
};

//
// Kakadu specific implementation
//
void set_default_colour_weights(kdu_params *siz);

// Factory function: see declaration in llimagej2c.cpp
LLImageJ2CImpl* fallbackCreateLLImageJ2CImpl()
{
	return new LLImageJ2CKDU();
}

std::string LLImageJ2CKDU::getEngineInfo() const
{
	return llformat("KDU %s", KDU_CORE_VERSION);
}

class LLKDUDecodeState
{
public:
	LLKDUDecodeState(kdu_tile tile, kdu_byte *buf, S32 row_gap,
					 kdu_codestream* codestreamp);
	~LLKDUDecodeState();
	bool processTileDecode(F32 decode_time, bool limit_time = true);

private:
	S32 mNumComponents;
	bool mUseYCC;
	kdu_dims mDims;
	kdu_sample_allocator mAllocator;
	kdu_push_pull_params mPushPullParams;
	kdu_tile_comp mComps[4];
	kdu_line_buf mLines[4];
	kdu_pull_ifc mEngines[4];
	bool mReversible[4]; // Some components may be reversible and others not
	int mBitDepths[4];   // Original bit-depth may be quite different from 8
	
	kdu_tile mTile;
	kdu_byte *mBuf;
	S32 mRowGap;
};

// Stuff for new kdu error handling
class LLKDUMessage: public kdu_message
{
public:
	LLKDUMessage(const std::string& type):
		mType(type)
	{}

	virtual void put_text(const char *s)
	{
		LL_INFOS() << "KDU " << mType << ": " << s << LL_ENDL;
	}

	virtual void put_text(const kdu_uint16 *s)
	{
		// The previous implementation simply streamed 's' to the log. So
		// either this put_text() override was never called -- or it produced
		// some baffling log messages -- because I assert that streaming a
		// const kdu_uint16* to a std::ostream will display only the hex value
		// of the pointer.
		LL_INFOS() << "KDU " << mType << ": "
				   << utf16str_to_utf8str(llutf16string(s)) << LL_ENDL;
	}

private:
	std::string mType;
};

struct LLKDUMessageWarning : public LLKDUMessage
{
	LLKDUMessageWarning():
		LLKDUMessage("Warning")
	{
		kdu_customize_warnings(this);
	}
};
// Instantiating LLKDUMessageWarning calls kdu_customize_warnings() with the
// new instance. Make it static so this only happens once.
static LLKDUMessageWarning sWarningHandler;

struct LLKDUMessageError : public LLKDUMessage
{
	LLKDUMessageError():
		LLKDUMessage("Error")
	{
		kdu_customize_errors(this);
	}

	virtual void flush(bool end_of_message = false)
	{
		// According to the documentation nat found:
		// http://pirlwww.lpl.arizona.edu/resources/guide/software/Kakadu/html_pages/globals__kdu$mize_errors.html
		// "If a kdu_error object is destroyed, handler→flush will be called with
		// an end_of_message argument equal to true and the process will
		// subsequently be terminated through exit. The termination may be
		// avoided, however, by throwing an exception from within the message
		// terminating handler→flush call."
		// So throwing an exception here isn't arbitrary: we MUST throw an
		// exception if we want to recover from a KDU error.
		// Because this confused me: the above quote specifically refers to
		// the kdu_error class, which is constructed internally within KDU at
		// the point where a fatal error is discovered and reported. It is NOT
		// talking about the kdu_message subclass passed to
		// kdu_customize_errors(). Destroying this static object at program
		// shutdown will NOT engage the behavior described above.
		if (end_of_message) 
		{
			LLTHROW(KDUError("LLKDUMessageError::flush()"));
		}
	}
};
// Instantiating LLKDUMessageError calls kdu_customize_errors() with the new
// instance. Make it static so this only happens once.
static LLKDUMessageError sErrorHandler;

LLImageJ2CKDU::LLImageJ2CKDU() : LLImageJ2CImpl(),
	mInputp(),
	mCodeStreamp(),
	mTPosp(),
	mTileIndicesp(),
	mRawImagep(NULL),
	mDecodeState(),
	mBlocksSize(-1),
	mPrecinctsSize(-1),
	mLevels(0)
{
}

LLImageJ2CKDU::~LLImageJ2CKDU()
{
	cleanupCodeStream(); // in case destroyed before decode completed
}

// Stuff for new simple decode
void transfer_bytes(kdu_byte *dest, kdu_line_buf &src, int gap, int precision);

// This is called by the real (private) initDecode() (keep_codestream true)
// and getMetadata() methods (keep_codestream false). As far as nat can tell,
// mode is always MODE_FAST. It was called by findDiscardLevelsBoundaries()
// as well, when that still existed, with keep_codestream true and MODE_FAST.
void LLImageJ2CKDU::setupCodeStream(LLImageJ2C &base, bool keep_codestream, ECodeStreamMode mode)
{
	S32 data_size = base.getDataSize();
	S32 max_bytes = (base.getMaxBytes() ? base.getMaxBytes() : data_size);

	//
	//  Initialization
	//
	mCodeStreamp.reset();

	// It's not clear to nat under what circumstances we would reuse a
	// pre-existing LLKDUMemSource instance. As of 2016-08-05, it consists of
	// two U32s and a pointer, so it's not as if it would be a huge overhead
	// to allocate a new one every time.
	// Also -- why is base.getData() tested specifically here? If that returns
	// NULL, shouldn't we bail out of the whole method?
	if (!mInputp && base.getData())
	{
		// The compressed data has been loaded
		// Setup the source for the codestream
		mInputp.reset(new LLKDUMemSource(base.getData(), data_size));
	}

	if (mInputp)
	{
		// This is LLKDUMemSource::reset(), not std::unique_ptr::reset().
		mInputp->reset();
	}

	mCodeStreamp->create(mInputp.get());

	// Set the maximum number of bytes to use from the codestream
	// *TODO: This seems to be wrong. The base class should have no idea of
	// how j2c compression works so no good way of computing what's the byte
	// range to be used.
	mCodeStreamp->set_max_bytes(max_bytes,true);

	//	If you want to flip or rotate the image for some reason, change
	// the resolution, or identify a restricted region of interest, this is
	// the place to do it.  You may use "kdu_codestream::change_appearance"
	// and "kdu_codestream::apply_input_restrictions" for this purpose.
	//	If you wish to truncate the code-stream prior to decompression, you
	// may use "kdu_codestream::set_max_bytes".
	//	If you wish to retain all compressed data so that the material
	// can be decompressed multiple times, possibly with different appearance
	// parameters, you should call "kdu_codestream::set_persistent" here.
	//	There are a variety of other features which must be enabled at
	// this point if you want to take advantage of them.  See the
	// descriptions appearing with the "kdu_codestream" interface functions
	// in "kdu_compressed.h" for an itemized account of these capabilities.

	switch (mode)
	{
	case MODE_FAST:
		mCodeStreamp->set_fast();
		break;
	case MODE_RESILIENT:
		mCodeStreamp->set_resilient();
		break;
	case MODE_FUSSY:
		mCodeStreamp->set_fussy();
		break;
	default:
		llassert(0);
		mCodeStreamp->set_fast();
	}

	kdu_dims dims;
	mCodeStreamp->get_dims(0,dims);

	S32 components = mCodeStreamp->get_num_components();

	// Check that components have consistent dimensions (for PPM file)
	for (int idx = 1; idx < components; ++idx)
	{
		kdu_dims other_dims;
		mCodeStreamp->get_dims(idx, other_dims);
		if (other_dims != dims)
		{
			// This method is only called from methods that catch KDUError.
			// We want to fail the image load, not crash the viewer.
			LLTHROW(KDUError(STRINGIZE("Component " << idx << " dimensions "
									   << stringize(other_dims)
									   << " do not match component 0 dimensions "
									   << stringize(dims) << "!")));
		}
	}

	// Get the number of resolution levels in that image
	mLevels = mCodeStreamp->get_min_dwt_levels();
	
	// Set the base dimensions
	base.setSize(dims.size.x, dims.size.y, components);
	base.setLevels(mLevels);
	
	if (!keep_codestream)
	{
		mCodeStreamp.reset();
		mInputp.reset();
	}
}

void LLImageJ2CKDU::cleanupCodeStream()
{
	mInputp.reset();
	mDecodeState.reset();
	mCodeStreamp.reset();
	mTPosp.reset();
	mTileIndicesp.reset();
}

// This is the protected virtual method called by LLImageJ2C::initDecode().
// However, as far as nat can tell, LLImageJ2C::initDecode() is called only by
// llimage_libtest.cpp's load_image() function. No detectable production use.
bool LLImageJ2CKDU::initDecode(LLImageJ2C &base, LLImageRaw &raw_image, int discard_level, int* region)
{
	return initDecode(base,raw_image,0.0f,MODE_FAST,0,4,discard_level,region);
}

bool LLImageJ2CKDU::initEncode(LLImageJ2C &base, LLImageRaw &raw_image, int blocks_size, int precincts_size, int levels)
{
	mPrecinctsSize = precincts_size;
	if (mPrecinctsSize != -1)
	{
		mPrecinctsSize = get_lower_power_two(mPrecinctsSize,MAX_PRECINCT_SIZE);
		mPrecinctsSize = llmax(mPrecinctsSize,MIN_PRECINCT_SIZE);
	}
	mBlocksSize = blocks_size;
	if (mBlocksSize != -1)
	{
		mBlocksSize = get_lower_power_two(mBlocksSize,MAX_BLOCK_SIZE);
		mBlocksSize = llmax(mBlocksSize,MIN_BLOCK_SIZE);
		if (mPrecinctsSize != -1)
		{
			mBlocksSize = llmin(mBlocksSize,mPrecinctsSize);	// blocks *must* be smaller than precincts
		}
	}
	mLevels = levels;
	if (mLevels != 0)
	{
		mLevels = llclamp(mLevels,MIN_DECOMPOSITION_LEVELS,MAX_DECOMPOSITION_LEVELS);
		base.setLevels(mLevels);
	}
	return true;
}

// This is the real (private) initDecode() called both by the protected
// initDecode() method and by decodeImpl(). As far as nat can tell, only the
// decodeImpl() usage matters for production.
bool LLImageJ2CKDU::initDecode(LLImageJ2C &base, LLImageRaw &raw_image, F32 decode_time, ECodeStreamMode mode, S32 first_channel, S32 max_channel_count, int discard_level, int* region)
{
	base.resetLastError();

	// *FIX: kdu calls our callback function if there's an error, and then bombs.
	// To regain control, we throw an exception, and catch it here.
	try
	{
		// Merov : Test!! DO NOT COMMIT!!
		//findDiscardLevelsBoundaries(base);

		base.updateRawDiscardLevel();
		setupCodeStream(base, true, mode);

		mRawImagep = &raw_image;
		mCodeStreamp->change_appearance(false, true, false);

		// Apply loading discard level and cropping if required
		kdu_dims* region_kdu = NULL;
		if (region != NULL)
		{
			region_kdu = new kdu_dims;
			region_kdu->pos.x  = region[0];
			region_kdu->pos.y  = region[1];
			region_kdu->size.x = region[2] - region[0];
			region_kdu->size.y = region[3] - region[1];
		}
		int discard = (discard_level != -1 ? discard_level : base.getRawDiscardLevel());
		//LL_INFOS() << "Merov debug : initDecode, discard used = " << discard << ", asked = " << discard_level << LL_ENDL;
		// Apply loading restrictions
		mCodeStreamp->apply_input_restrictions( first_channel, max_channel_count, discard, 0, region_kdu);
		
		// Clean-up
		if (region_kdu)
		{
			delete region_kdu;
			region_kdu = NULL;
		}

		// Resize raw_image according to the image to be decoded
		kdu_dims dims; mCodeStreamp->get_dims(0,dims);
		S32 channels = base.getComponents() - first_channel;
		channels = llmin(channels,max_channel_count);
		raw_image.resize(dims.size.x, dims.size.y, channels);

		if (!mTileIndicesp)
		{
			mTileIndicesp.reset(new kdu_dims);
		}
		mCodeStreamp->get_valid_tiles(*mTileIndicesp);
		if (!mTPosp)
		{
			mTPosp.reset(new kdu_coords);
			mTPosp->y = 0;
			mTPosp->x = 0;
		}
	}
	catch (const KDUError& msg)
	{
		base.setLastError(msg.what());
		return false;
	}
	catch (const kdu_exception& kdu_value)
	{
		// KDU internally throws kdu_exception. It's possible that such an
		// exception might leak out into our code. Catch kdu_exception
		// specially because boost::current_exception_diagnostic_information()
		// could do nothing with it.
		base.setLastError(report_kdu_exception(kdu_value));
		return false;
	}
	catch (...)
	{
		base.setLastError("Unknown J2C error: " +
						  boost::current_exception_diagnostic_information());
		return false;
	}

	return true;
}


// Returns true to mean done, whether successful or not.
bool LLImageJ2CKDU::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decode_time, S32 first_channel, S32 max_channel_count)
{
	ECodeStreamMode mode = MODE_FAST;

	LLTimer decode_timer;

	if (!mCodeStreamp->exists())
	{
		if (!initDecode(base, raw_image, decode_time, mode, first_channel, max_channel_count))
		{
			// Initializing the J2C decode failed, bail out.
			cleanupCodeStream();
			return true; // done
		}
	}

	// These can probably be grabbed from what's saved in the class.
	kdu_dims dims;
	mCodeStreamp->get_dims(0,dims);

	// Now we are ready to walk through the tiles processing them one-by-one.
	kdu_byte *buffer = raw_image.getData();
	if (!buffer)
	{
		base.setLastError("Memory error");
		base.decodeFailed();
		cleanupCodeStream();
		return true; // done
	}

	while (mTPosp->y < mTileIndicesp->size.y)
	{
		while (mTPosp->x < mTileIndicesp->size.x)
		{
			try
			{
				if (!mDecodeState)
				{
					kdu_tile tile = mCodeStreamp->open_tile(*(mTPosp)+mTileIndicesp->pos);

					// Find the region of the buffer occupied by this
					// tile.  Note that we have no control over
					// sub-sampling factors which might have been used
					// during compression and so it can happen that tiles
					// (at the image component level) actually have
					// different dimensions.  For this reason, we cannot
					// figure out the buffer region occupied by a tile
					// directly from the tile indices.  Instead, we query
					// the highest resolution of the first tile-component
					// concerning its location and size on the canvas --
					// the `dims' object already holds the location and
					// size of the entire image component on the same
					// canvas coordinate system.  Comparing the two tells
					// us where the current tile is in the buffer.
					S32 channels = base.getComponents() - first_channel;
					if (channels > max_channel_count)
					{
						channels = max_channel_count;
					}
					kdu_resolution res = tile.access_component(0).access_resolution();
					kdu_dims tile_dims; res.get_dims(tile_dims);
					kdu_coords offset = tile_dims.pos - dims.pos;
					int row_gap = channels*dims.size.x; // inter-row separation
					kdu_byte *buf = buffer + offset.y*row_gap + offset.x*channels;
					mDecodeState.reset(new LLKDUDecodeState(tile, buf, row_gap,
															mCodeStreamp.get()));
				}
				// Do the actual processing
				F32 remaining_time = decode_time - decode_timer.getElapsedTimeF32();
				// This is where we do the actual decode.  If we run out of time, return false.
				if (mDecodeState->processTileDecode(remaining_time, (decode_time > 0.0f)))
				{
					mDecodeState.reset();
				}
				else
				{
					// Not finished decoding yet.
					//					setLastError("Ran out of time while decoding");
					return false;
				}
			}
			catch (const KDUError& msg)
			{
				base.setLastError(msg.what());
				base.decodeFailed();
				cleanupCodeStream();
				return true; // done
			}
			catch (const kdu_exception& kdu_value)
			{
				// KDU internally throws kdu_exception. It's possible that such an
				// exception might leak out into our code. Catch kdu_exception
				// specially because boost::current_exception_diagnostic_information()
				// could do nothing with it.
				base.setLastError(report_kdu_exception(kdu_value));
				base.decodeFailed();
				cleanupCodeStream();
				return true; // done
			}
			catch (...)
			{
				base.setLastError("Unknown J2C error: " +
								  boost::current_exception_diagnostic_information());
				base.decodeFailed();
				cleanupCodeStream();
				return true; // done
			}


			mTPosp->x++;
		}
		mTPosp->y++;
		mTPosp->x = 0;
	}

	cleanupCodeStream();

	return true;
}


bool LLImageJ2CKDU::encodeImpl(LLImageJ2C &base, const LLImageRaw &raw_image, const char* comment_text, F32 encode_time, bool reversible)
{
	// Declare and set simple arguments
	bool transpose = false;
	bool vflip = true;
	bool hflip = false;

	try
	{
		// Set up input image files
		siz_params siz;
		
		// Should set rate someplace here
		LLKDUMemIn mem_in(raw_image.getData(),
			raw_image.getDataSize(),
			raw_image.getWidth(),
			raw_image.getHeight(),
			raw_image.getComponents(),
			&siz);

		base.setSize(raw_image.getWidth(), raw_image.getHeight(), raw_image.getComponents());

		int num_components = raw_image.getComponents();

		siz.set(Scomponents,0,0,num_components);
		siz.set(Sdims,0,0,base.getHeight());  // Height of first image component
		siz.set(Sdims,0,1,base.getWidth());   // Width of first image component
		siz.set(Sprecision,0,0,8);  // Image samples have original bit-depth of 8
		siz.set(Ssigned,0,0,false); // Image samples are originally unsigned

		kdu_params *siz_ref = &siz; 
		siz_ref->finalize();
		siz_params transformed_siz; // Use this one to construct code-stream
		transformed_siz.copy_from(&siz,-1,-1,-1,0,transpose,false,false);

		// Construct the `kdu_codestream' object and parse all remaining arguments
		U32 max_output_size = base.getWidth()*base.getHeight()*base.getComponents();
		max_output_size = (max_output_size < 1000 ? 1000 : max_output_size);
		U8 *output_buffer = new U8[max_output_size];
		U32 output_size = 0; // Address updated by LLKDUMemTarget to give the final compressed buffer size
		LLKDUMemTarget output(output_buffer, output_size, max_output_size);

		kdu_codestream codestream;
		codestream.create(&transformed_siz,&output);

		if (comment_text)
		{
			// Set the comments for the codestream
			kdu_codestream_comment comment = codestream.add_comment();
			comment.put_text(comment_text);
		}

		if (num_components >= 3)
		{
			// Note that we always use YCC and not YUV
			// *TODO: Verify this doesn't screws up reversible textures (like sculpties) as YCC is not reversible but YUV is...
			set_default_colour_weights(codestream.access_siz());
		}

		// Set codestream options
		int nb_layers = 0;
		kdu_long layer_bytes[MAX_NB_LAYERS];
		U32 max_bytes = (U32)(base.getWidth() * base.getHeight() * base.getComponents());

		// Rate is the argument passed into the LLImageJ2C which specifies the target compression rate. The default is 8:1.
		// *TODO: mRate is actually always 8:1 in the viewer. Test different values.
		llassert (base.mRate > 0.f);
		max_bytes = (U32)((F32)(max_bytes) * base.mRate);
		
		// This code is where we specify the target number of bytes for each quality layer.
		// We're using a logarithmic spacing rule that fits with our way of fetching texture data.
		// Note: For more info on this layers business, read kdu_codestream::flush() doc in kdu_compressed.h
		layer_bytes[nb_layers++] = FIRST_PACKET_SIZE;
		U32 i = MIN_LAYER_SIZE;
		while ((i < max_bytes) && (nb_layers < (MAX_NB_LAYERS-1)))
		{
			layer_bytes[nb_layers++] = i;
			i *= 4;
		}
		// Note: for small images, we can have (max_bytes < FIRST_PACKET_SIZE), hence the test
		if (layer_bytes[nb_layers-1] < max_bytes)
		{
			// Set the last quality layer so to fit the preset compression ratio
			layer_bytes[nb_layers++] = max_bytes;
		}

		if (reversible)
		{
			// Use 0 for a last quality layer for reversible images so all remaining code blocks will be flushed
			// Hack: KDU encoding for reversible images has a bug for small images that leads to j2c images that 
			// cannot be open or are very blurry. Avoiding that last layer prevents the problem to happen.
			if ((base.getWidth() >= 32) || (base.getHeight() >= 32))
			{
				layer_bytes[nb_layers++] = 0;
			}
			codestream.access_siz()->parse_string("Creversible=yes");
			// *TODO: we should use yuv in reversible mode
			// Don't turn this on now though as it creates problems on decoding for the moment
			//codestream.access_siz()->parse_string("Cycc=no");
		}
		
		std::string layer_string = llformat("Clayers=%d",nb_layers);
		codestream.access_siz()->parse_string(layer_string.c_str());
		
		// Set up data ordering, markers, etc... if precincts or blocks specified
		if ((mBlocksSize != -1) || (mPrecinctsSize != -1))
		{
			if (mPrecinctsSize != -1)
			{
				std::string precincts_string = llformat("Cprecincts={%d,%d}",mPrecinctsSize,mPrecinctsSize);
				codestream.access_siz()->parse_string(precincts_string.c_str());
			}
			if (mBlocksSize != -1)
			{
				std::string blocks_string = llformat("Cblk={%d,%d}",mBlocksSize,mBlocksSize);
				codestream.access_siz()->parse_string(blocks_string.c_str());
			}
			std::string ordering_string = llformat("Corder=LRCP");
			codestream.access_siz()->parse_string(ordering_string.c_str());
			std::string PLT_string = llformat("ORGgen_plt=yes");
			codestream.access_siz()->parse_string(PLT_string.c_str());
			std::string Parts_string = llformat("ORGtparts=R");
			codestream.access_siz()->parse_string(Parts_string.c_str());
		}
		
		// Set the number of wavelets subresolutions (aka levels) 
		if (mLevels != 0)
		{
			std::string levels_string = llformat("Clevels=%d",mLevels);
			codestream.access_siz()->parse_string(levels_string.c_str());
		}
		
		// Complete the encode settings
		codestream.access_siz()->finalize_all();
		codestream.change_appearance(transpose,vflip,hflip);

		// Now we are ready for sample data processing
		kdc_flow_control *tile = new kdc_flow_control(&mem_in,codestream);
		bool done = false;
		while (!done)
		{ 
			// Process line by line
			if (tile->advance_components())
			{
				tile->process_components();
			}
			else
			{
				done = true;
			}
		}

		// Produce the compressed output
		codestream.flush(layer_bytes,nb_layers);

		// Cleanup
		delete tile;
		codestream.destroy();

		// Now that we're done encoding, create the new data buffer for the compressed
		// image and stick it there.
		base.copyData(output_buffer, output_size);
		base.updateData(); // set width, height
		delete[] output_buffer;
	}
	catch(const KDUError& msg)
	{
		base.setLastError(msg.what());
		return false;
	}
	catch (const kdu_exception& kdu_value)
	{
		// KDU internally throws kdu_exception. It's possible that such an
		// exception might leak out into our code. Catch kdu_exception
		// specially because boost::current_exception_diagnostic_information()
		// could do nothing with it.
		base.setLastError(report_kdu_exception(kdu_value));
		return false;
	}
	catch( ... )
	{
		base.setLastError("Unknown J2C error: " +
						  boost::current_exception_diagnostic_information());
		return false;
	}

	return true;
}

bool LLImageJ2CKDU::getMetadata(LLImageJ2C &base)
{
	// *FIX: kdu calls our callback function if there's an error, and
	// then bombs. To regain control, we throw an exception, and
	// catch it here.
	try
	{
		setupCodeStream(base, false, MODE_FAST);
		return true;
	}
	catch (const KDUError& msg)
	{
		base.setLastError(msg.what());
		return false;
	}
	catch (const kdu_exception& kdu_value)
	{
		// KDU internally throws kdu_exception. It's possible that such an
		// exception might leak out into our code. Catch kdu_exception
		// specially because boost::current_exception_diagnostic_information()
		// could do nothing with it.
		base.setLastError(report_kdu_exception(kdu_value));
		return false;
	}
	catch (...)
	{
		base.setLastError("Unknown J2C error: " +
						  boost::current_exception_diagnostic_information());
		return false;
	}
}

/*****************************************************************************/
/* STATIC                        copy_block                                  */
/*****************************************************************************/

/*==========================================================================*|
// Only called by copy_tile(), which is itself commented out
static void copy_block(kdu_block *in, kdu_block *out)
{
	if (in->K_max_prime != out->K_max_prime)
    { 
		std::cout << "Cannot copy blocks belonging to subbands with different quantization parameters." << std::endl; 
		return;
	}
	if ((in->size.x != out->size.x) || (in->size.y != out->size.y))  
    { 
		std::cout << "Cannot copy code-blocks with different dimensions." << std::endl; 
		return;
	}
	out->missing_msbs = in->missing_msbs;
	if (out->max_passes < (in->num_passes+2))        // Gives us enough to round up
		out->set_max_passes(in->num_passes+2,false); // to the next whole bit-plane
	out->num_passes = in->num_passes;
	int num_bytes = 0;
	for (int z=0; z < in->num_passes; z++)
    {
		num_bytes += (out->pass_lengths[z] = in->pass_lengths[z]);
		out->pass_slopes[z] = in->pass_slopes[z];
    }
	
    // Just copy compressed code-bytes. Block transcoding not supported.
	if (out->max_bytes < num_bytes)
		out->set_max_bytes(num_bytes,false);
	memcpy(out->byte_buffer,in->byte_buffer,(size_t) num_bytes);
}
|*==========================================================================*/

/*****************************************************************************/
/* STATIC                        copy_tile                                   */
/*****************************************************************************/

/*==========================================================================*|
// Only called by findDiscardLevelsBoundaries(), which is itself commented out
static void
copy_tile(kdu_tile tile_in, kdu_tile tile_out, int tnum_in, int tnum_out,
		  kdu_params *siz_in, kdu_params *siz_out, int skip_components,
		  int &num_blocks)
{
	int num_components = tile_out.get_num_components();
	int new_tpart=0, next_tpart = 1;
	
	for (int c=0; c < num_components; c++)
    {
		kdu_tile_comp comp_in, comp_out;
		comp_in = tile_in.access_component(c);
		comp_out = tile_out.access_component(c);
		int num_resolutions = comp_out.get_num_resolutions();
		//std::cout << "    Copying tile : num_resolutions = " << num_resolutions << std::endl;
		for (int r=0; r < num_resolutions; r++)
        {
			kdu_resolution res_in;  res_in = comp_in.access_resolution(r);
			kdu_resolution res_out; res_out = comp_out.access_resolution(r);
			int b, min_band;
			int num_bands = res_in.get_valid_band_indices(min_band);
			std::cout << "        Copying tile : num_bands = " << num_bands << std::endl;
			for (b=min_band; num_bands > 0; num_bands--, b++)
            {
				kdu_subband band_in;  band_in = res_in.access_subband(b);
				kdu_subband band_out; band_out = res_out.access_subband(b);
				kdu_dims blocks_in;  band_in.get_valid_blocks(blocks_in);
				kdu_dims blocks_out; band_out.get_valid_blocks(blocks_out);
				if ((blocks_in.size.x != blocks_out.size.x) ||
					(blocks_in.size.y != blocks_out.size.y))
                { 
					std::cout << "Transcoding operation cannot proceed: Code-block partitions for the input and output code-streams do not agree." << std::endl;
					return;
				}
				kdu_coords idx;
				//std::cout << "            Copying tile : block indices, x = " << blocks_out.size.x << " and y = " << blocks_out.size.y << std::endl;
				for (idx.y=0; idx.y < blocks_out.size.y; idx.y++)
				{
					for (idx.x=0; idx.x < blocks_out.size.x; idx.x++)
					{
						kdu_block *in =
						band_in.open_block(idx+blocks_in.pos,&new_tpart);
						for (; next_tpart <= new_tpart; next_tpart++)
							siz_out->copy_from(siz_in,tnum_in,tnum_out,next_tpart,
											   skip_components);
						kdu_block *out = band_out.open_block(idx+blocks_out.pos);
						copy_block(in,out);
						band_in.close_block(in);
						band_out.close_block(out);
						num_blocks++;
					}
				}
            }
        }
    }
}
|*==========================================================================*/

// Find the block boundary for each discard level in the input image.
// We parse the input blocks and copy them in a temporary output stream.
// For the moment, we do nothing more that parsing the raw list of blocks and outputing result.
/*==========================================================================*|
// See comments in header file for why this is commented out.
void LLImageJ2CKDU::findDiscardLevelsBoundaries(LLImageJ2C &base)
{
	// We need the number of levels in that image before starting.
	getMetadata(base);
	
	for (int discard_level = 0; discard_level < mLevels; discard_level++)
	{
		//std::cout << "Parsing discard level = " << discard_level << std::endl;
		// Create the input codestream object.
		setupCodeStream(base, true, MODE_FAST);
		mCodeStreamp->apply_input_restrictions(0, 4, discard_level, 0, NULL);
		mCodeStreamp->set_max_bytes(KDU_LONG_MAX,true);
		siz_params *siz_in = mCodeStreamp->access_siz();
	
		// Create the output codestream object.
		siz_params siz;
		siz.copy_from(siz_in,-1,-1,-1,0,discard_level,false,false,false);
		siz.set(Scomponents,0,0,mCodeStreamp->get_num_components());
	
		U32 max_output_size = base.getWidth()*base.getHeight()*base.getComponents();
		max_output_size = (max_output_size < 1000 ? 1000 : max_output_size);
		U8 *output_buffer = new U8[max_output_size];
		U32 output_size = 0; // Address updated by LLKDUMemTarget to give the final compressed buffer size
		LLKDUMemTarget output(output_buffer, output_size, max_output_size);
		kdu_codestream codestream_out; 
		codestream_out.create(&siz,&output);
		//codestream_out.share_buffering(*mCodeStreamp);
		siz_params *siz_out = codestream_out.access_siz();
		siz_out->copy_from(siz_in,-1,-1,-1,0,discard_level,false,false,false);
		codestream_out.access_siz()->finalize_all(-1);
	
		// Set up rate control variables
		kdu_long max_bytes = KDU_LONG_MAX;
		kdu_params *cod = siz_out->access_cluster(COD_params);
		int total_layers;  cod->get(Clayers,0,0,total_layers);
		kdu_long *layer_bytes = new kdu_long[total_layers];
		int nel, non_empty_layers = 0;
	
		// Now ready to perform the transfer of compressed data between streams
		int flush_counter = INT_MAX;
		kdu_dims tile_indices_in;  
		mCodeStreamp->get_valid_tiles(tile_indices_in);
		kdu_dims tile_indices_out;