Problems with receiving messages in UDP client (pingpong)

I have a problem with completing a UDP client in a Ping/Pong software, measuring the RTTs. The client seems to establish a connection with the server (using in a first time a TCP connection), but when it comes to sending the message and receiving an answer it fails, not receiving datagrams even after some attemps. I have to implement a non blocking recv with timeout. I'll leave you the entire do_ping function: My solution is:

/*
* This function sends and waits for a reply on a socket.
* char message[]: message to send
* size_t msg_size: message length
*/
double do_ping(size_t msg_size, int msg_no, char message[msg_size], int ping_socket, double timeout)
{
   int lost_count = 0, recv_errno;
   char answer_buffer[msg_size];
   ssize_t recv_bytes, sent_bytes;
   struct timespec send_time, recv_time;
   double roundtrip_time_ms;
   int re_try = 0;

   /*** write msg_no at the beginning of the message buffer ***/
   /*** TO BE DONE START ***/

   sprintf(message, "%d\n", msg_no);

   /*** TO BE DONE END ***/

   do {
       debug(" ... sending message %d\n", msg_no);
   /*** Store the current time in send_time ***/
   /*** TO BE DONE START ***/

   if(clock_gettime(CLOCK_REALTIME, &send_time) != 0)
       fail_errno("clock_gettime");

   /*** TO BE DONE END ***/

   /*** Send the message through the socket (non blocking mode) ***/
   /*** TO BE DONE START ***/

   sent_bytes = send(ping_socket, message, msg_size, MSG_DONTWAIT);
   if(sent_bytes != msg_size && errno != EAGAIN && errno != EWOULDBLOCK)
       fail_errno("send failure");

   /*** TO BE DONE END ***/

I think that the problem is this section (timeout is 3000 ms in double):

   /*** Receive answer through the socket (non blocking mode, with timeout) ***/
   /*** TO BE DONE START ***/

   struct timeval to;
   to.tv_sec = (long) (timeout*pow(10, -3));
   to.tv_usec = 0;

   if(setsockopt(ping_socket, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)) != 0)
       fail_errno("setsockopt timeout");

   for (int offset = 0; (offset + (recv_bytes = recv(ping_socket, answer_buffer + offset, sent_bytes - offset, MSG_WAITALL))) < msg_size; offset += recv_bytes){
        debug(" ... received %zd bytes back\n", recv_bytes);
        if (recv_bytes < 0)
            fail_errno("Error receiving data");
   }
    
    
   /*** TO BE DONE END **/

This Is the end of the section.

   /*** Store the current time in recv_time ***/
   /*** TO BE DONE START ***/

   if(clock_gettime(CLOCK_REALTIME, &recv_time) != 0)
       fail_errno("clock_gettime");

   /*** TO BE DONE END ***/

    roundtrip_time_ms = timespec_delta2milliseconds(&recv_time, &send_time);

    while ( recv_bytes < 0 && (recv_errno == EAGAIN || recv_errno == EWOULDBLOCK)
                && roundtrip_time_ms < timeout ) {
        recv_bytes = recv(ping_socket, answer_buffer, sizeof(answer_buffer), 0);
                    recv_errno = errno;
                    if ( recv_bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK )
                fail_errno("UDP ping could not recv from UDP socket");
                    
            if (clock_gettime(CLOCK_TYPE, &recv_time))
            fail_errno("Cannot get receive-time");
        roundtrip_time_ms = timespec_delta2milliseconds(&recv_time, &send_time);
                    sscanf(answer_buffer,"%ld %ld, %ld %ld\n",
                                            &(recv_time.tv_sec), &(recv_time.tv_nsec),
                                            &(send_time.tv_sec), &(send_time.tv_nsec));
                roundtrip_time_ms -= timespec_delta2milliseconds(&send_time, &recv_time);
    }
    if (recv_bytes < sent_bytes) {  /*time-out elapsed: packet was lost */
        lost_count++;
        if (recv_bytes < 0)
            recv_bytes = 0;
        printf("\n ... received %zd bytes instead of %zd (lost count = %d); re-trying ...\n", recv_bytes, sent_bytes, lost_count);
        if (++re_try > MAXUDPRESEND) {
            printf(" ... giving-up!\n");
            fail("too many lost datagrams");
        }
        printf(" ... re-trying ...\n");
    }
} while (sent_bytes != recv_bytes);

return roundtrip_time_ms;
}

Thank you so much for the help and for the time invested in this.