#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/poll.h>
#include <sys/time.h>
#include <fcntl.h>
#include <netdb.h>
#include <malloc.h>
#include <errno.h>
#include <popt.h>
static int server;
struct poptOption poptOpts[]=
{
{
"server",
's',
POPT_ARG_NONE,
NULL,
's',
"Enable the server",
""
},
POPT_AUTOHELP
{
NULL,
0,
0,
NULL,
0
}
};
void
do_server(int port)
{
int fd;
char buffer[2048];
struct sockaddr addr;
struct sockaddr_in ipaddr;
socklen_t addrlen;
size_t len;
int rv;
fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (fd == -1) {
perror("Could not bind to port");
exit(1);
}
ipaddr.sin_family = AF_INET;
ipaddr.sin_port = htons(port);
ipaddr.sin_addr.s_addr = INADDR_ANY;
rv = bind(fd, (struct sockaddr *) &ipaddr, sizeof(ipaddr));
if (rv) {
perror("Could not bind to port");
exit(1);
}
for (;;) {
addrlen = sizeof(addr);
len = recvfrom(fd, buffer, sizeof(buffer), 0, &addr, &addrlen);
if (len > 0) {
rv = sendto(fd, buffer, len, 0, &addr, addrlen);
if (rv == -1)
perror("error in sendto");
} else {
perror("Error in recvfrom");
}
}
}
static long
diff_timeval(struct timeval *left,
struct timeval *right)
{
if ( (left->tv_sec < right->tv_sec)
|| ( (left->tv_sec == right->tv_sec)
&& (left->tv_usec < right->tv_usec)))
{
/* If left < right, just force to zero, don't allow negative
numbers. */
return 0;
}
return (((left->tv_sec - right->tv_sec) * 1000000)
+ (left->tv_usec - right->tv_usec));
}
static long
average(long *data, int size, long *max, long *min)
{
int i;
long rv = 0;
*max = LONG_MIN;
*min = LONG_MAX;
for (i=0; i<size; i++) {
if (data[i] < 0)
continue;
if (data[i] > *max)
*max = data[i];
if (data[i] < *min)
*min = data[i];
rv += data[i];
}
return (rv / size);
}
void do_client(struct in_addr *dest_addr, int port, int count, int size)
{
int fd;
struct sockaddr_in ipaddr;
struct sockaddr addr;
socklen_t addrlen;
int rv;
int i;
size_t len;
char buffer[2048];
char buffer2[2048];
long *times;
long avg, max, min;
int curr_port;
times = malloc(sizeof(long) * count);
if (!times) {
fprintf(stderr, "Out of memory\n");
exit(1);
}
fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (fd == -1) {
fprintf(stderr, "Out of memory\n");
exit(1);
}
curr_port = 1000;
do {
curr_port++;
ipaddr.sin_family = AF_INET;
ipaddr.sin_port = htons(curr_port);
ipaddr.sin_addr.s_addr = INADDR_ANY;
rv = bind(fd, (struct sockaddr *) &ipaddr, sizeof(ipaddr));
} while ((curr_port < 65536) && (rv == -1));
ipaddr.sin_family = AF_INET;
ipaddr.sin_port = htons(port);
ipaddr.sin_addr = *dest_addr;
for (i=0; i<size; i++) {
buffer[i] = i;
}
for (i=0; i<count; i++) {
struct timeval start_time;
struct timeval end_time;
struct timeval wait_time;
fd_set read_set;
times[i] = -1;
FD_ZERO(&read_set);
FD_SET(fd, &read_set);
wait_time.tv_sec = 2;
wait_time.tv_usec = 0;
gettimeofday(&start_time, NULL);
rv = sendto(fd, buffer, size, 0,
(struct sockaddr *) &ipaddr, sizeof(ipaddr));
if (rv == -1) {
perror("Error in sendto");
continue;
}
retry:
rv = select(fd+1, &read_set, NULL, NULL, &wait_time);
gettimeofday(&end_time, NULL);
if (rv == -1) {
if (errno == EINTR)
goto retry;
perror("Error in select");
continue;
}
if (rv == 0) {
fprintf(stderr, "Timeout waiting for response %d\n", i);
continue;
}
addrlen = sizeof(addr);
len = recvfrom(fd, buffer2, sizeof(buffer2), 0, &addr, &addrlen);
if (rv == -1) {
perror("Error in recvfrom");
continue;
}
if (len != size) {
fprintf(stderr, "Invalid length in response to buffer %d,"
" waiting more\n", i);
goto retry;
}
if (memcmp(buffer, buffer2, size) != 0) {
fprintf(stderr, "Invalid data in response to buffer %d,"
" waiting more\n", i);
goto retry;
}
times[i] = diff_timeval(&end_time, &start_time);
}
avg = average(times, count, &max, &min);
printf("Average: %ldus, Max: %ldus, Min: %ldus\n", avg, max, min);
}
int
main(int argc, const char *argv[])
{
int o;
struct hostent *ent;
struct in_addr addr;
int port;
int count;
int size;
poptContext poptCtx = poptGetContext("ip_lat", argc, argv, poptOpts,0);
while (( o = poptGetNextOpt(poptCtx)) >= 0)
{
switch( o )
{
case 's':
server = 1;
break;
default:
poptPrintUsage(poptCtx, stderr, 0);
exit(1);
}
}
argv = poptGetArgs(poptCtx);
if (!argv) {
fprintf(stderr, "Not enough arguments\n");
exit(1);
}
for (argc=0; argv[argc]!= NULL; argc++)
;
if ((server && (argc < 1)) || (!server && (argc < 4))) {
fprintf(stderr, "Not enough arguments\n");
exit(1);
}
if (server) {
port = atoi(argv[0]);
do_server(port);
} else {
ent = gethostbyname(argv[0]);
if (!ent) {
fprintf(stderr, "gethostbyname failed: %s\n", strerror(h_errno));
exit(1);
}
memcpy(&addr, ent->h_addr_list[0], ent->h_length);
port = atoi(argv[1]);
count = atoi(argv[2]);
size = atoi(argv[3]);
do_client(&addr, port, count, size);
}
return 0;
}
> I've attached a small program to measure latency of
> round-trip time on UDP. If I send 85-byte packets between
> two of my machines, I get 170us round-trip latency. If I
> send 86-byte packets, I get 1329us latency.
> This seems quite odd. If I test on the eepro100 driver, I
> get expected linear increase in round-trip time as the packet
> size increases, and it never gets close to 1300us.
This sounds like a side-effect of the "CPU Cycle Saver" feature to
bundle Rx packets per one interrupt. See
Documentation/networking/e100.txt. I haven't tried your setup, but I
would guess that you can play around with the BundleSmallFr module
parameter, or better yet, if you want the lowest latencies, turn off CPU
Saver (ucode=0).
CPU Saver trades latency for reduced interrupts, resulting in CPU
savings, hence the name.
Hope this helps.
-scott
sounds like the cpu cycle saver
check out networking/e100.txt
On Thu, 2003-05-15 at 15:54, Corey Minyard wrote:
> I'm seeing an odd thing with the e100 driver. It seems to be this way
> with the 2.4 series and with 2.5.68, and I couldn't find anything with a
> search.
>
> I've attached a small program to measure latency of round-trip time on
> UDP. If I send 85-byte packets between two of my machines, I get 170us
> round-trip latency. If I send 86-byte packets, I get 1329us latency.
> This seems quite odd. If I test on the eepro100 driver, I get expected
> linear increase in round-trip time as the packet size increases, and it
> never gets close to 1300us.
>
> To run the program, do:
>
> ./ip_lat -s <port>
>
> on one machine to run the server, and then do
>
> ./ip_lat <server IP> <port> 1000 85
>
> to send 1000 85-byte packets from another machine. Change the 85 to 86
> to see the latency go up.
>
> -Corey
>
> ______________________________________________________________________
>
> #include <sys/types.h>
> #include <sys/socket.h>
> #include <netinet/in.h>
> #include <sys/poll.h>
> #include <sys/time.h>
> #include <fcntl.h>
> #include <netdb.h>
> #include <malloc.h>
> #include <errno.h>
> #include <popt.h>
>
> static int server;
>
> struct poptOption poptOpts[]=
> {
> {
> "server",
> 's',
> POPT_ARG_NONE,
> NULL,
> 's',
> "Enable the server",
> ""
> },
> POPT_AUTOHELP
> {
> NULL,
> 0,
> 0,
> NULL,
> 0
> }
> };
>
> void
> do_server(int port)
> {
> int fd;
> char buffer[2048];
> struct sockaddr addr;
> struct sockaddr_in ipaddr;
> socklen_t addrlen;
> size_t len;
> int rv;
>
> fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
> if (fd == -1) {
> perror("Could not bind to port");
> exit(1);
> }
>
> ipaddr.sin_family = AF_INET;
> ipaddr.sin_port = htons(port);
> ipaddr.sin_addr.s_addr = INADDR_ANY;
>
> rv = bind(fd, (struct sockaddr *) &ipaddr, sizeof(ipaddr));
> if (rv) {
> perror("Could not bind to port");
> exit(1);
> }
>
> for (;;) {
> addrlen = sizeof(addr);
> len = recvfrom(fd, buffer, sizeof(buffer), 0, &addr, &addrlen);
> if (len > 0) {
> rv = sendto(fd, buffer, len, 0, &addr, addrlen);
> if (rv == -1)
> perror("error in sendto");
> } else {
> perror("Error in recvfrom");
> }
> }
> }
>
> static long
> diff_timeval(struct timeval *left,
> struct timeval *right)
> {
> if ( (left->tv_sec < right->tv_sec)
> || ( (left->tv_sec == right->tv_sec)
> && (left->tv_usec < right->tv_usec)))
> {
> /* If left < right, just force to zero, don't allow negative
> numbers. */
> return 0;
> }
>
> return (((left->tv_sec - right->tv_sec) * 1000000)
> + (left->tv_usec - right->tv_usec));
> }
>
> static long
> average(long *data, int size, long *max, long *min)
> {
> int i;
> long rv = 0;
>
> *max = LONG_MIN;
> *min = LONG_MAX;
> for (i=0; i<size; i++) {
> if (data[i] < 0)
> continue;
> if (data[i] > *max)
> *max = data[i];
> if (data[i] < *min)
> *min = data[i];
> rv += data[i];
> }
>
> return (rv / size);
> }
>
> void do_client(struct in_addr *dest_addr, int port, int count, int size)
> {
> int fd;
> struct sockaddr_in ipaddr;
> struct sockaddr addr;
> socklen_t addrlen;
> int rv;
> int i;
> size_t len;
> char buffer[2048];
> char buffer2[2048];
> long *times;
> long avg, max, min;
> int curr_port;
>
> times = malloc(sizeof(long) * count);
> if (!times) {
> fprintf(stderr, "Out of memory\n");
> exit(1);
> }
>
> fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
> if (fd == -1) {
> fprintf(stderr, "Out of memory\n");
> exit(1);
> }
>
> curr_port = 1000;
> do {
> curr_port++;
> ipaddr.sin_family = AF_INET;
> ipaddr.sin_port = htons(curr_port);
> ipaddr.sin_addr.s_addr = INADDR_ANY;
>
> rv = bind(fd, (struct sockaddr *) &ipaddr, sizeof(ipaddr));
> } while ((curr_port < 65536) && (rv == -1));
>
> ipaddr.sin_family = AF_INET;
> ipaddr.sin_port = htons(port);
> ipaddr.sin_addr = *dest_addr;
>
> for (i=0; i<size; i++) {
> buffer[i] = i;
> }
>
> for (i=0; i<count; i++) {
> struct timeval start_time;
> struct timeval end_time;
> struct timeval wait_time;
> fd_set read_set;
>
> times[i] = -1;
>
> FD_ZERO(&read_set);
> FD_SET(fd, &read_set);
> wait_time.tv_sec = 2;
> wait_time.tv_usec = 0;
>
> gettimeofday(&start_time, NULL);
> rv = sendto(fd, buffer, size, 0,
> (struct sockaddr *) &ipaddr, sizeof(ipaddr));
> if (rv == -1) {
> perror("Error in sendto");
> continue;
> }
> retry:
> rv = select(fd+1, &read_set, NULL, NULL, &wait_time);
> gettimeofday(&end_time, NULL);
> if (rv == -1) {
> if (errno == EINTR)
> goto retry;
> perror("Error in select");
> continue;
> }
> if (rv == 0) {
> fprintf(stderr, "Timeout waiting for response %d\n", i);
> continue;
> }
>
> addrlen = sizeof(addr);
> len = recvfrom(fd, buffer2, sizeof(buffer2), 0, &addr, &addrlen);
> if (rv == -1) {
> perror("Error in recvfrom");
> continue;
> }
>
> if (len != size) {
> fprintf(stderr, "Invalid length in response to buffer %d,"
> " waiting more\n", i);
> goto retry;
> }
> if (memcmp(buffer, buffer2, size) != 0) {
> fprintf(stderr, "Invalid data in response to buffer %d,"
> " waiting more\n", i);
> goto retry;
> }
>
> times[i] = diff_timeval(&end_time, &start_time);
> }
>
> avg = average(times, count, &max, &min);
> printf("Average: %ldus, Max: %ldus, Min: %ldus\n", avg, max, min);
> }
>
> int
> main(int argc, const char *argv[])
> {
> int o;
> struct hostent *ent;
> struct in_addr addr;
> int port;
> int count;
> int size;
>
> poptContext poptCtx = poptGetContext("ip_lat", argc, argv, poptOpts,0);
>
> while (( o = poptGetNextOpt(poptCtx)) >= 0)
> {
> switch( o )
> {
> case 's':
> server = 1;
> break;
>
> default:
> poptPrintUsage(poptCtx, stderr, 0);
> exit(1);
> }
> }
>
> argv = poptGetArgs(poptCtx);
>
> if (!argv) {
> fprintf(stderr, "Not enough arguments\n");
> exit(1);
> }
>
> for (argc=0; argv[argc]!= NULL; argc++)
> ;
>
> if ((server && (argc < 1)) || (!server && (argc < 4))) {
> fprintf(stderr, "Not enough arguments\n");
> exit(1);
> }
>
> if (server) {
> port = atoi(argv[0]);
> do_server(port);
> } else {
> ent = gethostbyname(argv[0]);
> if (!ent) {
> fprintf(stderr, "gethostbyname failed: %s\n", strerror(h_errno));
> exit(1);
> }
> memcpy(&addr, ent->h_addr_list[0], ent->h_length);
> port = atoi(argv[1]);
>
> count = atoi(argv[2]);
> size = atoi(argv[3]);
> do_client(&addr, port, count, size);
> }
>
> return 0;
> }
On Thu, 2003-05-15 at 10:31, Jonathan Brown wrote:
> sounds like the cpu cycle saver
>
> check out networking/e100.txt
I had seen something similar in an application of mine. If I loaded the
e100 module with BundleMax=1 I got the same results as using the
eepro100 driver.
Mark.
>
> On Thu, 2003-05-15 at 15:54, Corey Minyard wrote:
> > I'm seeing an odd thing with the e100 driver. It seems to be this way
> > with the 2.4 series and with 2.5.68, and I couldn't find anything with a
> > search.
> >
> > I've attached a small program to measure latency of round-trip time on
> > UDP. If I send 85-byte packets between two of my machines, I get 170us
> > round-trip latency. If I send 86-byte packets, I get 1329us latency.
> > This seems quite odd. If I test on the eepro100 driver, I get expected
> > linear increase in round-trip time as the packet size increases, and it
> > never gets close to 1300us.
> >
> > To run the program, do:
> >
> > ./ip_lat -s <port>
> >
> > on one machine to run the server, and then do
> >
> > ./ip_lat <server IP> <port> 1000 85
> >
> > to send 1000 85-byte packets from another machine. Change the 85 to 86
> > to see the latency go up.
> >
> > -Corey
--
Mark Haverkamp <[email protected]>
Thanks for the response.
I looked at the code, and there doesn't seem to be a way to turn it off
dynamically or with a bootline option. I have to have the driver
compiled into the kernel (because I'm netbooting with NFS) so I can't
really use a module parameter.
A dynamic field in /proc would be quite nice.
-Corey
Feldman, Scott wrote:
>>I've attached a small program to measure latency of
>>round-trip time on UDP. If I send 85-byte packets between
>>two of my machines, I get 170us round-trip latency. If I
>>send 86-byte packets, I get 1329us latency.
>>This seems quite odd. If I test on the eepro100 driver, I
>>get expected linear increase in round-trip time as the packet
>>size increases, and it never gets close to 1300us.
>>
>>
>
>This sounds like a side-effect of the "CPU Cycle Saver" feature to
>bundle Rx packets per one interrupt. See
>Documentation/networking/e100.txt. I haven't tried your setup, but I
>would guess that you can play around with the BundleSmallFr module
>parameter, or better yet, if you want the lowest latencies, turn off CPU
>Saver (ucode=0).
>
>CPU Saver trades latency for reduced interrupts, resulting in CPU
>savings, hence the name.
>
>Hope this helps.
>
>-scott
>
>
>
>