C#

...
...
//deflate and chunk
object[] outData = DeflateAndChunk(data);
...
...
//dechunk and inflate
string reconstructedData = DechunkAndInflate(outData);
...
...

private static byte[] DeChunkData(object[] baseData)
{
	int returnLength = 0;
	foreach(byte[] ba in baseData)
	{
		returnLength += ba.Length;
	}

	byte[] readBuffer = new byte[returnLength];
	using (Stream outStream = new MemoryStream(readBuffer))
	{
		
		for (int loop=0; loop<baseData.Length; loop++)
		{
			byte[] dataChunk = (byte[])baseData[loop];
			outStream.Write(dataChunk, 0, dataChunk.Length);	
			baseData[loop] = null; //null of the bytes that are no longer needed, every little helps
		}


		outStream.Flush();
		outStream.Close();
	}

	GC.Collect();

	return readBuffer;
}

private static object[] ChunkData(byte[] baseData, int chunkSize)
{
	int returnArraySize = baseData.Length / chunkSize;
	int baseDataModulus = baseData.Length % chunkSize;
	if (baseDataModulus > 0)
	{
		returnArraySize++;
	}
	object[] returnData = new object[returnArraySize];

	using (Stream baseDataStream = new MemoryStream(baseData))
	{
		baseData = null;

		for (int loop=0;loop < returnArraySize; loop++)
		{
			int workingChunkSize = chunkSize;

			if (loop == returnArraySize)
			{
				workingChunkSize = baseDataModulus;
			}

			byte[] buffer = new byte[workingChunkSize];
			baseDataStream.Read(buffer, 0, workingChunkSize);
			returnData[loop] = buffer;
		}

		baseDataStream.Flush();
		baseDataStream.Close();
	}

	GC.Collect();

	return returnData;
}

public static object[] DeflateAndChunk(string inputMessage)
{
	//compress data and get checksum
	long adler32Checksum = 0;
	byte[] compressedData = CompressData(inputMessage, out adler32Checksum);

	inputMessage = null;
	
	//TODO - BEGIN - Define chunk sizes (config? Based on data size?)
	int chunkSize = 1000;
	//TODO - END Define chunk sizes (config? Based on data size?)

	//chunk data based on params
	object[] chunkedData = ChunkData(compressedData, chunkSize);

	//TODO - BEGIN - thread off and send chunks using config from WS call
	//WS calls here, least of my worries until memory
	//usage for chunking and compression is sorted
	//TODO - END - thread off and send chunks using config from WS call

	//method only returns so it can be tested
	return chunkedData;
}

public static string DechunkAndInflate(object[] inputData)
{
	byte[] deChunkedData = DeChunkData(inputData);

	byte[] inflatedData = UnCompressData(deChunkedData);
	
	deChunkedData = null;

	return System.Text.ASCIIEncoding.ASCII.GetString(inflatedData);
}

private static byte[] CompressData(string message, out long adler32Checksum)
{
	byte[] returnData = null;

	//create base streams
	using (Stream inStream = new MemoryStream(System.Text.ASCIIEncoding.ASCII.GetBytes(message)))
	{
		message = null;

		using (Stream outStream = new MemoryStream())
		using (ZipStreams.DeflaterOutputStream deflatorStream = new ZipStreams.DeflaterOutputStream(outStream, 
			new ZipCompression.Deflater(ZipCompression.Deflater.BEST_SPEED)))
		{

			....
			//standard sharpzip code for compression
			....
		}
	}

	//calculate checksums
	BaseCompression.Checksums.Adler32 ad32 = new BaseCompression.Checksums.Adler32();
	ad32.Update(returnData);
	adler32Checksum = ad32.Value;

	//force garbage collection
	GC.Collect();

	return returnData;
}

private static byte[] UnCompressData(byte[] deflatedData)
{
	byte[] returnData = null;

	//create inflator streams
	using (Stream inStream = new MemoryStream(deflatedData))
	{
		deflatedData = null;

		using (MemoryStream outStream = new MemoryStream())
		using (ZipStreams.InflaterInputStream inflatorStream = new ZipStreams.InflaterInputStream(inStream))
		{
			....
			//standard sharpzip code for uncompression
			....
		}
	}

	//force garbage collection
	GC.Collect();

	//return
	return returnData;
}