UDP transmissions using select() longer time in windows than in linux

See more:

C++

Windows

Linux

I made 2000 transmissions from client to server with pauses between them. The 2000 pauses takes 6 seconds.

I tested my code in windows and takes 17 seconds in the 2000 transmissions
In linux it takes 6.2-6.4 seconds (depends if blocking/non blocking)

The question is: why it takes a lot more time in windows than in linux?

This is the receiving and sending functions:

void c_client::send_block(char *data_block, int size)
{
	int stat;
	stat = sendto(client_socket, data_block, size, 0, (struct sockaddr *) &si_other, sizeof(si_other)); //sends to server!!
	if (stat == -1)
	{
		char line[256] = "### c_client::send_block failed:"; strcat(line, strerror(errno)); 
		cout_mtx.lock(); cout << line << endl; cout_mtx.unlock();
	}
}
int c_client::receive_block(char *data_block)
{
	si_length = sizeof(si_other);
	int stat1 = -1;

	fd_set tmp = select_fd_set0;
	int ret = select((int)client_socket + 1, &tmp, NULL, NULL, &select_timeval);
	switch (ret)
	{
	case 0: return 0;
	case -1: cout_mtx.lock(); cout << "WARNING AT c_client::receive_block() making select()" << endl; cout_mtx.unlock(); return -1;
	default: 
		stat1 = (int) recvfrom(client_socket, data_block, UDP1_BUFFER_LONG, 0, (struct sockaddr *) &si_other, &si_length);
	}
	return stat1;//if -1 no data in!
}

CAUTION-1:
Some people returns the recvfrom() as size_t that is completely wrong because if results fails (-1) it can not be seen as long as size_t is unsigned

CAUTION-2:
To properly makes work this program (and any other using sockets) it must be performed this to allow increase the socket memory:

sudo sysctl -w net.core.rmem_max=2304000

What I have tried:

Here is the full header code header (working in linux and windows): Protocol.hpp

#ifndef  __PROTOCOLO_HPP
#define  __PROTOCOLO_HPP
/*
Version V000: Bien 2000 paquetes ok. Tarda lo mismo con TIMEOUT_US=1-500us si bien tarda 20-28s en mandar los 2000 paquetes debiendo ser algo mas de 6 segundos
V001: No select():
	  non-blocking     los 2000 paquetes son OK pero va lento: 84 segundos en vez de 28 con select().
	  En modo blocking los 2000 paquetes son OK (en init_server y init _client unsigned long mode=0) pero tarda 106 segundos
V002: Con select y non-blocking: va bien pero tarda un 15% mas que en modo blocking
V003: Se puede seleccionar entre blocking/nonblocking en linux y windows
      Modo blocking: tarda 16 segundos en mandar los 2000 paquetes (en modos release y debug x64)
	  Modo nonblocking: Tarda 18 segundos pero pierde el servidor 8 de 2000 paquetes
	  Funciona en linux, tarda 6.37 segundos en modo blocking y en modo nonblocking 6.43 pero se encalla un poco para salir.

CAUTION!!: Increase receiver socket in Linux that is 8k long only do  :

    sudo sysctl -w net.core.rmem_max=2304000

	Compilation linux:

	gcc  -std=c++11 -lstdc++ -pthread -o protocol_exe Protocol_V003.cpp Protocol_V003.hpp
*/


#include <iostream> //cout
#include <stdio.h>  //getchar()
#include <stdlib.h> //srand
#include <string.h> //memcpy
#include <thread>   //threads, usleep
#include <chrono>	//usleep
#include <mutex>          // std::mutex

#ifdef __linux__ 
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>    //usleep()
#include <sys/ioctl.h>
#define _BIND bind
#define TCPID_SERVER "10.130.20.187"  //linux. TCP_ID of the client. Used at the c_serverclass at send() and recvfrom()    MODIFY THIS!!!!!!!!!!!!!!!!!!!!!
#define _CLOSE close


#else
#include <winsock2.h>
#define snprintf _snprintf
#pragma comment(lib, "Ws2_32.lib")
#pragma warning(disable:4996)     //to allow using linux standard compatible functions
#define _BIND ::bind
//internal:"127.0.0.1" net A:  10.121.18.193  net B:10:130:20:187
#define TCPID_SERVER "10.121.18.193"   //TCP_ID of the server. Used at the c_client class   MODIFY THIS!!!!!!!!!!!!!!!!!!!!!
#define _CLOSE closesocket
#endif

using namespace std;
typedef unsigned char uchar;
typedef unsigned short ushort;

//=========================== The UDP zone =================================================================
#define BLOCK_HEADER 0xBEADFEAC   //ADA1DEAD
#define UDP1_BUFFER_LONG 62000  //max UDP=65507. (1280*720+10)/15=61440.66 usamos 61441 y asi se aprovecha mejor
#define PORT_NUM   2450			//PORT for listening data, used by both
#define TIMEOUT_US 100			//TIMEOUT to be used in select()
#define CMD_SENT 2000			//Max number of c_client commands to be sent for testing purposes
//#define NONBLOCKING				//undef for making blocking. It is 15% slower in nonblocking!!!!!!!!!!!

void sleep_us(long microseconds){ this_thread::sleep_for(std::chrono::microseconds(microseconds)); }

uchar message1_long[UDP1_BUFFER_LONG], message2_long[UDP1_BUFFER_LONG];
uchar message1_short[100], message2_short[100];
void fill_messages();
std::mutex cout_mtx;

class c_server
{
public:
	c_server(); //no tcp_id needed at the server side!
	~c_server();
	void server_run_state(); //main function. pixels4=number of pixels (that are divided by 4 before being sent)
	void set_end_state(bool state){ flag_end_server = state; }//if false makes server_run_state() to exit!!
private:
	//socket variables:
#ifdef __linux__ 
	socklen_t si_length;
	int server_socket;//server side
	int clientToServer_socket; //client side
#else
	int si_length;
	SOCKET server_socket; //server side
	//SOCKET clientToServer_socket; //client side
#endif
	struct sockaddr_in  anyclient;
	//the select side!
	fd_set select_fd_set0;//select_fd_set0 is initiated once again server socket, then select_fd_set=select_fd_set0 every time select() is called
	struct timeval select_timeval;

	//private functions:
	void init_server();
	void send_block(char *data_block, int size);
	int receive_block(char *data_block);

	uchar state123;
	static bool flag_end_server;
};


class c_client
{
public:
	c_client(const char *tcp_id){ init_client(tcp_id); }
	~c_client() { }
	void client_run_state();
private:
	void init_client(const char *tcp_id);
	void send_block(char *data_block, int size);
	int receive_block(char *data_block);
	
	
	//private data to be used in sockets:
	struct sockaddr_in si_other;//this is the sockaddr_in needed at the client side to send to server
	//struct sockaddr_in si_client;//this is the sockaddr_in used to receivefrom server
#ifdef __linux__ 
	socklen_t si_length;
	int client_socket; //client side
#else
	int si_length;
	SOCKET client_socket; //Connected to server tcpid & port. Used at recvfrom() and sendto()
#endif
	//the select side!
	fd_set select_fd_set0;//to be initiated!
	struct timeval select_timeval;
};
#endif

And this is the Protocol.cpp file:

#include "Protocol_V003.hpp"

void fill_messages()
{ 
	for (int i = 0; i < UDP1_BUFFER_LONG; i++)
	{ 
		message1_long[i] = rand() % 256; message2_long[i] = rand() % 256; 
		if (i < 100) 
		{ 
			message1_short[i] = rand() % 256; message2_short[i] = rand() % 256; 
		} 
	} 
	message1_long[0] =  1;//message1 long
	message2_long[0] =  2;//message2 long
	message1_short[0] = 3;//message1 short
	message2_short[0] = 4;//message2 short

	message1_long[1] = 1;//RUN
	message2_long[1] = 1;//RUN
	message1_short[1] = 1;//RUN
	message2_short[1] = 1;//RUN
}


//================================== SERVER =================================
c_server::c_server()
{
	init_server();
}

void c_server::init_server()
{
	flag_end_server = false;
	state123 = 1;//RUN
	printf("initializing server\n");
	sockaddr_in server; //used in this function only to initiate socket
	memset((uchar*)&server, 0, sizeof(server));
#ifdef __linux__ 
	/*Fill in server's sockaddr_in*/
#define SOCKADDR struct sockaddr   
#else
	WSADATA wsaData;
	WORD wVersionRequested = MAKEWORD(2, 0);
	if (WSAStartup(wVersionRequested, &wsaData) != 0)
	{
		cout << "c_server::init_server()  WSAStartup() error" << endl; getchar();
		return;
	}
#endif

	/*Fill in server's sockaddr_in*/
	server.sin_family = AF_INET;
	server.sin_addr.s_addr = INADDR_ANY;//inet_addr(tcp_id);
	server.sin_port = htons(PORT_NUM);

	memset((char *)&anyclient, 0, sizeof(anyclient));
	anyclient.sin_family = AF_INET;
	anyclient.sin_addr.s_addr = INADDR_ANY;
	anyclient.sin_port = htons(0000);//any client port!
	int slen = sizeof(anyclient);

	/*Create socket */
	server_socket = socket(AF_INET, SOCK_DGRAM, 0);
	if (server_socket == -1)
	{
		cout << "Error in c_server::init_server(): socket failed"; getchar();
		return;
	}
	//bind(server_socket, (SOCKADDR*)&server_address, sizeof(SOCKADDR));
	if (_BIND(server_socket, (SOCKADDR *)&server, sizeof(server)) == -1)
	{
		cout<<"Fail bind()"; getchar();
		_CLOSE(server_socket); return;
	}
	int buffsize = 24 * UDP1_BUFFER_LONG;
	//Increasing the socket buffer:
	if (setsockopt(server_socket, SOL_SOCKET, SO_RCVBUF, (char*)&buffsize, sizeof(int)) == -1)
	{
		cout << "Warning at c_server::init_server() could not increase receiver socket size" << endl; getchar();
	}
	else
		cout << "Increased server socket bufer size" << endl;

	//the select side!
	//fd_set select_fd_set;//to be initiated!
	//struct timeval select_timeval;
	select_timeval.tv_sec = 0;
	select_timeval.tv_usec = TIMEOUT_US;
	FD_ZERO(&select_fd_set0);
	FD_SET(server_socket, &select_fd_set0);

	//BLOCKING/NON BLOCKING:
#ifdef NONBLOCKING
#ifdef __linux__
	int opt = 1; ioctl(server_socket, FIONBIO, &opt);
#else
	unsigned long opt; ioctlsocket(server_socket, FIONBIO, &opt);
#endif
#endif

}

void c_server::send_block(char *data_block, int size)
{
	int stat;
	stat = sendto(server_socket, data_block, size, 0, (struct sockaddr *) &anyclient, sizeof(anyclient)); //sends to any client!!
	if (stat == -1)
	{
		char line[256] = "### c_server::send_block failed:"; strcat(line, strerror(errno));
		cout_mtx.lock(); cout << line << endl; cout_mtx.unlock();
	}
}
int c_server::receive_block(char *data_block)
{
	si_length = sizeof(anyclient);
	int stat1 = -1;

	fd_set tmp = select_fd_set0;
	int ret = select((int)server_socket + 1, &tmp, NULL, NULL, &select_timeval);    
	switch (ret)
	{
	case 0: return 0;
	case -1: cout_mtx.lock(); cout << "WARNING AT c_server::receive_block() making select()" << endl; cout_mtx.unlock(); return -1;
	default: 
		stat1 = (int) recvfrom(server_socket, data_block, UDP1_BUFFER_LONG, 0, (struct sockaddr *) &anyclient, &si_length);
	}
	return stat1;//if -1 no data in!
}

bool c_server::flag_end_server;

void c_server::server_run_state()
{
	uchar message[UDP1_BUFFER_LONG];
	ushort *num_cmd = (ushort *)&message[2];
	uchar *cmd = &message[1];
	int num_cmd_sent = 0, num_cmd_in = 0, num_cmd_err = 0;

	while (!flag_end_server)
	{
		int i1 = receive_block((char *)message);
		if (i1 > 0)
		{
			num_cmd_in++;
			cout_mtx.lock();
			cout << "server in cmd:" << (int) message[0] << " cmd_num=" << *num_cmd << endl;
			cout_mtx.unlock();

			//send any cmd:
			if (*num_cmd == 10) state123 = 2;
			else if (*num_cmd == 20) state123 = 3;
			else if (*num_cmd == 30) state123 = 1;

			ushort num_cmd1 = *num_cmd;
			if (num_cmd1 % 2 == 0)
				memcpy(message, message1_short, 100);
			else
				memcpy(message, message2_short, 100);
			*num_cmd = num_cmd1;
			*cmd = state123;
			send_block((char *)message, 100); 
			num_cmd_sent++;
			cout_mtx.lock();
			cout << "server out cmd:" << (int) message[0] << " cmd_num=" << *num_cmd << endl;
			if ((i1 != 100) && (i1 != UDP1_BUFFER_LONG))
			{
				cout << "====== ERROR sercer received command length=" << i1 << endl;
				num_cmd_err++;
			}
			cout_mtx.unlock();
		}
	}
	cout_mtx.lock();
	cout << "===== END c_server::server_run_state() =====" << endl;
	cout << "Server sent commands=" << num_cmd_sent << " received=" << num_cmd_in << " wrong commands=" << num_cmd_err << endl;
	cout_mtx.unlock();
}

//====================================== c_client =========================================================
void c_client::init_client(const char *tcp_id)
{
	fill_messages();//initializes messages to be sent for testing
	//initiating socket:
#ifdef __linux__
	//client_socket = socket(AF_INET, SOCK_DGRAM, 0);
	if ((client_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1)
	{
		printf("client socket() failed"); getchar();
		exit(EXIT_FAILURE);
	}
#else
	WSADATA wsaData;
	WORD wVersionRequested = MAKEWORD(2, 0);
	if (WSAStartup(wVersionRequested, &wsaData) != 0)
	{
		cout << "c_udp::init_client() WSAStartup() error" << endl; getchar();
		return;
	}

	//create socket
	if ((client_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1)
	{
		printf("client socket() failed with error code : %d", WSAGetLastError());
		exit(EXIT_FAILURE);
	}
#endif
	/*Create client socket*/
	if (client_socket == -1)
	{
		cout << "Warning: client socket not created trying again:" << endl; int i1 = 1;
		if (setsockopt(client_socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&i1, sizeof(int)) < 0)
			perror("setsockopt(SO_REUSEADDR) failed");
	}

	/*Fill in client's sockaddr_in */
	memset(&si_other, 0, sizeof(si_other));
	si_other.sin_family = AF_INET;
	si_other.sin_port = htons(PORT_NUM);
	si_other.sin_addr.s_addr = inet_addr(tcp_id); // |= htonl(0x1ff);

	select_timeval.tv_sec = 0;
	select_timeval.tv_usec = TIMEOUT_US;
	FD_ZERO(&select_fd_set0);
	FD_SET(client_socket, &select_fd_set0);
	cout << "Connected c_client with server network ip: " << tcp_id << " Port=" << PORT_NUM << endl;

#ifdef NONBLOCKING
#ifdef __linux__
	int opt = 1;ioctl(client_socket, FIONBIO, &opt);
#else
	unsigned long opt; ioctlsocket(client_socket, FIONBIO, &opt);
#endif
#endif
}




void c_client::send_block(char *data_block, int size)
{
	int stat;
	stat = sendto(client_socket, data_block, size, 0, (struct sockaddr *) &si_other, sizeof(si_other)); //sends to server!!
	if (stat == -1)
	{
		char line[256] = "### c_client::send_block failed:"; strcat(line, strerror(errno)); 
		cout_mtx.lock(); cout << line << endl; cout_mtx.unlock();
	}
}
int c_client::receive_block(char *data_block)
{
	si_length = sizeof(si_other);
	int stat1 = -1;

	fd_set tmp = select_fd_set0;
	int ret = select((int)client_socket + 1, &tmp, NULL, NULL, &select_timeval);
	switch (ret)
	{
	case 0: return 0;
	case -1: cout_mtx.lock(); cout << "WARNING AT c_client::receive_block() making select()" << endl; cout_mtx.unlock(); return -1;
	default: 
		stat1 = (int) recvfrom(client_socket, data_block, UDP1_BUFFER_LONG, 0, (struct sockaddr *) &si_other, &si_length);
	}
	return stat1;//if -1 no data in!
}


void c_client::client_run_state()
{
	uchar message[UDP1_BUFFER_LONG];
	ushort *num_cmd = (ushort *)&message[2];
	int num_cmd_sent = 0, num_cmd_in = 0, num_cmd_err = 0;

	for (ushort i = 0; i < CMD_SENT; i++)
	{
		if (i % 8 == 0)
		{
			if ((i / 8) % 2 == 1)
				memcpy(message, message1_long, UDP1_BUFFER_LONG);
			else
				memcpy(message, message2_long, UDP1_BUFFER_LONG);
			*num_cmd = i;
			send_block((char *)message, UDP1_BUFFER_LONG);
			num_cmd_sent++;
			sleep_us(10000);
		}
		else
		{
			if (i % 2 == 1)
				memcpy(message, message1_short, 100);
			else
				memcpy(message, message2_short, 100);
			*num_cmd = i;
			send_block((char *)message, 100);
			num_cmd_sent++;
			sleep_us(2000);
		}
		cout_mtx.lock();
		cout << "client out cmd:" << (int) message[0] << " cmd_num=" << *num_cmd << endl;
		cout_mtx.unlock();

		int i1 = receive_block((char *)message);
		if (i1 > 0)
		{
			num_cmd_in++;
			cout_mtx.lock();
			cout << "client in cmd:" << (int)message[0] << " cmd_num=" << *num_cmd << endl;
			if ((i1 != 100) && (i1 != UDP1_BUFFER_LONG))
			{
				cout << "====== ERROR c_client received command length=" << i1 << endl;
				num_cmd_err++;
			}
			cout_mtx.unlock();
		}
	}
	cout_mtx.lock();
	cout << "===== END c_client::server_run_state() =====" << endl;
	cout << "Client sent commands=" << num_cmd_sent << " received=" << num_cmd_in << " wrong commands=" << num_cmd_err << endl;
	cout_mtx.unlock();
}


int main()
{
	c_client *client = new c_client(TCPID_SERVER);
	c_server *server = new c_server();
	//sending the server thread!!!
	cout << "Running threads" << endl;
	time_t ini, fin;

	ini = clock();
	thread th_server(&c_server::server_run_state, server);
	sleep_us(10000);
	thread th_client(&c_client::client_run_state, client);
	th_client.join();
	sleep_us(2000);//2ms
	server->set_end_state(true);
	th_server.join();
	fin = clock();
	cout << "\n============= END =============" << endl;
	cout << "Total time=" << 1.0*(fin - ini) / CLOCKS_PER_SEC << " seconds" << endl;
	cout << "Expected>=" << CMD_SENT / 8 * 10e-3 + (CMD_SENT - CMD_SENT / 8)*2e-3 << " seconds" << endl;
	getchar();
	return 1;
}