How can I do my code Optimal?

Although RLE is used (or optional) with a few well known image formats, unless you're using posterized images, there are better schemes for file storage. RLE only considers consecutive values during compression, it can easily miss opportunities and may even increase image size, e.g. a pixel level checkerboard pattern will expand under RLE while LZ77 could deflate up to 99%.

That said, it can be good to implement things like this for one's own edification.

I have some old code lying around for RLE compression, and although it was designed to work with a graphics engine, the compressor emits a byte stream, so it shouldn't be difficult to adapt. Anyway, here's some advice and some code.

1. Decide on a format and enforce it using structures.
2. RLE is easier to implement in terms of a scanline than an entire image.
3. Build up complex functions from smaller less complex ones.
4. Keep things as simple as possible.

Each scanline begins with a header specifying pitch (byte length) for that line; successive headers specify a skip/copy pair for each run, followed by copy bytes worth of data.

Snippet header,

struct rlehead {
/* rle header type. */
	static const unsigned int size;

	rlehead() : pitch(0) {}
	explicit rlehead(const void* in);

	void read(const void* in);
	void write(void* out) const;

	union {
		unsigned int pitch;
		struct { unsigned short skip, copy; };
	};
};
const unsigned int rlehead::size = sizeof(rlehead);

inline rlehead::rlehead(const void* in)
{	this->read(in);
}

inline void rlehead::read(const void* in)
{	*this = *static_cast<const>(in);
}

inline void rlehead::write(void* out) const
{	*static_cast<rlehead*>(out) = *this;
}
</const>

Snippet encode,

inline char* rle_advance_scanline_(char* x, int count = 1)
/* advance rle scanline along y-axis. */
{
	while ( count-- )
	 {
	   rlehead head(x);
	   x += head.pitch;
	 }
	return x;
}

void rle_encode_eval_line_(Surface* surf, int y, std::vector<rlehead>& list)
/* generate headers from pixel data. */
{
	const PixelBase& op = Pixel.op();

	Color key = surf->col_key();
	int bpp = surf->bpp();
	int x = 0;

	while ( 1 )
	 {
	   rlehead head;
	   for ( ; x < surf->w() && op.get(surf, x, y) == key; x++ )
	      head.skip += bpp;
	   for ( ; x < surf->w() && op.get(surf, x, y) != key; x++ )
	      head.copy += bpp;
	   list.push_back(head);

	   if (head.copy == 0)
	      break;
	 }
}

void rle_encode_commit_line_(Surface* surf, int y,
  std::vector<rlehead>::iterator first, char* tmpbase)
/* write encoded line data to buffer. */
{
	char* pix = surf->pixels(0, y);
	char* tmp = tmpbase + rlehead::size;

	rlehead head;
	while ( 1 )
	 {
	   head = *first++;
	   head.write(tmp);
	   tmp += head.size;
	   pix += head.skip;

	   if (head.copy == 0)
	      break;
	   fwdcopy(tmp, pix, head.copy);
	   tmp += head.copy;
	   pix += head.copy;
	 }
	head.pitch = tmp - tmpbase;
	head.write(tmpbase);
}

void rle_encode_(Surface* surf)
/* encode surface. */
{
	const int bufsize = surf->h() * surf->pitch();
	char* tmpbase = new char[bufsize*2];
	char* tmp = tmpbase;

	std::auto_ptr<char> sentinal(tmpbase);

	for ( int y = 0; y < surf->h(); y++ )
	 {
	   std::vector<rlehead> list;
	   rle_encode_eval_line_(surf, y, list);
	   rle_encode_commit_line_(surf, y, list.begin(), tmp);
	   tmp = rle_advance_scanline_(tmp);

	   if (tmp - tmpbase > bufsize)
	      return;
	 }

	fwdcopy(surf->pixels(), tmpbase, tmp - tmpbase);
	surf->set_encoding(RLE_ENCODE);
}
</rlehead></char></rlehead></rlehead>

Snippet decode,

void std_encode_commit_line_(char* tmpbase, int y, Surface* surf)
/* write decoded line data to surface. */
{
	const PixelBase& op = Pixel.op();

	char* pix = surf->pixels(0, y);
	char* tmp = tmpbase + rlehead::size;

	while ( 1 )
	 {
	   rlehead head(tmp);
	   tmp += head.size;

	   op.hz_fill(pix, surf->col_key(), head.skip);
	   pix += head.skip;

	   if (head.copy == 0)
	      break;
	   fwdcopy(pix, tmp, head.copy);
	   pix += head.copy;
	   tmp += head.copy;
	 }
}

void std_encode_(Surface* surf)
/* decode rle surface. */
{
	char* pixbase = surf->pixels();
	char* pix = rle_advance_scanline_(pixbase, surf->h());

	const int bufsize = pix - pixbase;
	char* tmpbase = new char[bufsize];
	char* tmp = tmpbase;

	fwdcopy(tmpbase, surf->pixels(), bufsize);

	for ( int y = 0; y < surf->h(); y++ )
	 {
	   std_encode_commit_line_(tmp, y, surf);
	   tmp = rle_advance_scanline_(tmp);
	 }

	delete [] tmpbase;
	surf->set_encoding(STD_ENCODE);
}

Hope this helps.