Path: utzoo!utgpu!news-server.csri.toronto.edu!rpi!think.com!mintaka!ogicse!milton!uw-beaver!cornell!news
From: pat@cs.cornell.edu (Pat Stephenson)
Newsgroups: comp.sys.isis
Subject: Re: Performance benchmarks for isis
Message-ID: <1991May27.190805.2232@cs.cornell.edu>
Date: 27 May 91 19:08:05 GMT
Article-I.D.: cs.1991May27.190805.2232
References: <1991May15.230622.1378@digi.lonestar.org>
Sender: news@cs.cornell.edu (USENET news user)
Reply-To: pat@cs.cornell.edu (Pat Stephenson)
Organization: Cornell University Computer Science Department
Lines: 533
In-Reply-To: bwoster@digi.lonestar.org (Bill Woster)
Nntp-Posting-Host: turing4.cs.cornell.edu

I include below a small program that I use for testing isis message-passing
speed.  It can be operated in a variety of modes - one sender, more
than one sender, various numbers of replies expected, various packet
sizes, and so on.  A variant of this program was used to generate the
numbers in our forthcoming TOCS paper (available for ftp from
ftp.cs.cornell.edu, file pub/bypass.ps.Z).  HOWEVER, those numbers were
obtained running under ISISV3.0; those of you using earlier versions of
ISIS will not do as well.

USAGE:

You need to start up several copies manually - one for each group
member.  Startup is as follows:

1) First copy:
    perftest [options] [isis port]
2) subsequent copies:
    perftest [isis port]

Messages will be sent (as specified by the options) and any copy that
was sending messages will print statistics when finished.

There are probably better ways to write this program, but I just
needed the numbers.

The options are below.  All except "-i" and "-e" take an integer argument.

	-g Group size.  The total number of group members.

	-s Senders.  The number of group members that will send
        messages.

	-r Replies.  The number of replies expected to each message.
	Setting this to 0 results in messages being sent continuously
 	(streaming)

	-p Packet size.  The size of each message.

	-c Count.  The number of messages send by each sender.

	-l Measurement granularity.  Read the clock this often.
        Since message sending often takes less time than a clock tick,
        this option can be used to measure the average cost of message
        sends.  It's only used when RPC-style messages are being sent,
        i.e. replies are expected.

	-i Perform an isis dump on termination

	-q Warmup.  Send "Warmup" messages before starting to
           take measurements.

	-e Messages are copied to the sender.

	-d Discard.  Discard measurement outliers.  Value is % to
           discard at each end.

	-f Type.  Broadcast type.  0=cbcast, 1=fbcast, 2=abcast.

Program starts here.
---------------------------------------------------------------------
/*
 *	Test ISIS performance
 *      Based on an early skeleton by KB, but much modified by PFFS.
 */

#include <sys/time.h>
#include <sys/resource.h>
#include "isis.h"
#include "debug.h"

#define START_GRAN 30
#define NO_GRAN -1


/* isis entries */  

#define REPLY 1
#define DONE 2
#define STREAM_SYNC 3

/* GLOBAL STATE */

/* the total size of group we are dealing with */
int groupsize = 2;

/* # of people sending data.  The BOTTOM people in the group
   will send, unless we are doing ABCAST. */
int senders = 1;

/* total # of messages to send */
int msg_count = START_GRAN;

/* how often to measure time */
int granularity = NO_GRAN;

/* Size of each message to send.  */
int msg_size = 1000;

/* # of replies to make and wait for.  The TOP
   people in the group will make the replies, unless we are doing ABCAST */

int nreplies = 1;

/* type of broadcast to use */

#define CBCAST 0
#define FBCAST 1
#define ABCAST 2

int bcast_type = ABCAST;

/* dump at the end ? */

int dumpit = 0;

/* default is not to send to oneself */
int nonexclude = 0;

/* END OF GLOBAL STATE */

int reply_member;

/** do this many to warm up things */
int startup = 5;

int pn;

int discps = 5;

int gotargs = 0;
int myrank;
int port = 2043;

void test_reply();

/* Compute elapsed time, but convert to useconds */
struct timeval time_0;

init_elapsed_time() {
    gettimeofday(&time_0, (struct timezone*)0);
}

unsigned long
  elapsed_time() {
    struct timeval tp;
    int i;
    gettimeofday(&tp, (struct timezone*)0);
    i = (tp.tv_sec-time_0.tv_sec)*1000000 + tp.tv_usec-time_0.tv_usec;
    return i;
}


do_xfer (loc)
  int   loc;
{
    if (loc == -1)
        xfer_out (loc, "%d%d%d%d%d%d%d%d%d",
		  groupsize, senders, msg_count, msg_size, nreplies, bcast_type, dumpit, nonexclude,discps);
}

get_xfer (loc, mp)
  int       loc;
  message   *mp;
{
  msg_get (mp, "%d%d%d%d%d%d%d%d%d",
	    &groupsize, &senders, &msg_count, &msg_size, &nreplies, &bcast_type, &dumpit, &nonexclude, &discps);
  isis_entry(REPLY, test_reply, "test_reply");
  if (gotargs == 1)
    panic("gave arguments to more than one instantiation");
}


int
  parseargs(argc, argv)
char **argv;
{
  char c;
  extern char *optarg;
  extern int optind;
  init_elapsed_time();
  while ((c = getopt(argc, argv, "c:d:ef:g:il:p:q:r:s:")) != -1) {
    gotargs = 1;
      switch (c) {
      case 'c':
	msg_count = atoi(optarg);
	break;
      case 'd':
	discps = atoi(optarg);
	break;
      case 'e':
	nonexclude = 1;
	break;
      case 'f':
	bcast_type = atoi(optarg);
	break;
      case 'g':
	groupsize = atoi(optarg);
	break;
      case 'i':
	dumpit = 1;
	break;
      case 'l':
	granularity = atoi(optarg);
	break;
      case 'p':
	msg_size = atoi(optarg);
	break;
      case 'q':
	startup = atoi(optarg);
	break;
      case 'r':
	nreplies = atoi(optarg);
	break;
      case 's':
	senders = atoi(optarg);
	break;
      default:
      case '?':
	printf ("usage:\nperf\t -g size\n\t -s senders\n\t -c msg_count\n\t -p size\n\t -r replies\n\t -l granularity\n\t -f bcast type 0=c,1=f,2=a\n\t -d discard percent\n\t -i dump afterwards\n");
	fflush (stdout);
	exit(0);
      }
    }
  if (nreplies > groupsize)
    nreplies = groupsize;
  if (optind < argc) {
    port = atoi(argv[optind]);
  }
  if (granularity == NO_GRAN)
    granularity = msg_count;
  return gotargs;
}

main(argc, argv)
  char **argv;
  {
    void test_join();
    void test_done();
    void ssync();
    int i = parseargs(argc, argv);
    isis_init(port);
    if (i) {
      isis_entry(REPLY, test_reply, "test reply");
      isis_entry(STREAM_SYNC, ssync, "eos sync");
      isis_entry(DONE, test_done, "test_done");
      isis_mainloop(test_join, (void*)0);
    }
  }


address *gid;
address *mid;

void
test_join()
  {
    void change();
    gid = pg_join("new performance test",
		  PG_MONITOR, change, 0,
		  PG_XFER, 0, do_xfer, get_xfer, 0);
  }

#define	MAXLEN	15000

int int_compare(a, b)
  int *a, *b;
  {
	return(*a - *b);
  }


void change(gv)
  register groupview *gv;
  {
    void senddata();
    int cursize, i;

    if(!addr_isnull(&gv->gv_departed))
      exit(0);

    gid = &gv->gv_gaddr;
    cursize = gv->gv_nmemb;
    myrank = pg_rank (gid, &my_address);
    reply_member =
      (((bcast_type < ABCAST) && ( myrank >= (groupsize - nreplies))) ||
       ((bcast_type == ABCAST) && (myrank < nreplies)));
     if (cursize > groupsize) {
      isis_perror("too many people joined!");
    }
    if (cursize == groupsize) {
      if (((myrank < senders) && (bcast_type < ABCAST)) ||
	  ((myrank >= groupsize - senders) && (bcast_type == ABCAST)) )
	t_fork(senddata, (void*)0);
    }
  }

void test_done(mp)
     register message *mp;
{
  if (dumpit != 0)
    cl_dump(-1, "Dump");
  exit(0);
}

void
test_reply(mp)
     register message *mp;
{
  char *data; int *len, i;
  address *sender;
  if (reply_member) {
    reply_l("", mp,  "");
  }
}

void
ssync(mp)
  register message *mp;
  {
    char *data; int *len, i;
    address *sender;
    sender = msg_getsender(mp);
    i = pg_rank (gid, sender);
    reply_l("", mp,  "%C", &i, sizeof(int), 0);
  }

void senddata()
{
  int *delays = (int*)malloc(msg_count * sizeof(int));
  char **replyptr;
  char *buffer = malloc(msg_size);
  int *replysize;
  register unsigned start, i, finishup;
  register n, j;
  struct rusage rstart, rfinish;
  int min, max, sd, discard, samples = msg_count/granularity;
  double sum, sumsq, sqrt(), utime, stime, mean;
  address *target;
  char *format;
  message* m;

  format = ((nreplies == groupsize) || (nonexclude ==1)
		? "" : "x");
  target = gid;

  msg_count = samples*granularity;
  replyptr = (char**)malloc(groupsize*sizeof(char*));
  replysize = (int*)malloc(groupsize*sizeof(int*))
;
  m = msg_gen("%*C", buffer, msg_size, NULL, 0);

  switch (bcast_type ) {
  case FBCAST:
    for(i = 0; i < startup; i++) {
      if (fbcast_l(format, target, REPLY, "%*C", buffer, msg_size, 0,	   
		   nreplies, "%-C", &replyptr, &replysize) != nreplies) {
	panic("Unexpected fbcast rval");
      }
    }
    break;
  case CBCAST:
    for(i = 0; i < startup; i++) {
      if (cbcast_l(format, target, REPLY, "%*C", buffer, msg_size, 0,	   
		   nreplies, "%-C", &replyptr, &replysize) != nreplies) {
	panic("Unexpected cbcast rval");
      }
    }
    break;
  case ABCAST:
    for(i = 0; i < startup; i++) {
      if (abcast_l(format, target, REPLY, "%*C", buffer, msg_size, 0,	   
		   nreplies, "%-C", &replyptr, &replysize) != nreplies) {
	panic("Unexpected abcast rval");
      }
    }
    break;
  default:
    fprintf(stderr, "unknown bcast typ!\n");
  }
  getrusage(RUSAGE_SELF, &rstart);
  start = elapsed_time();

  switch (bcast_type) {
  case FBCAST:
    for(n = 0; n < samples; n++) {
      for(i = 0; i < granularity; i++) {
	if (fbcast_l(format, target, REPLY, "%*C", buffer, msg_size, 0,
		     nreplies, "") != nreplies) {
	  panic("Unexpected fbcast rval");
	}
      }
      i = elapsed_time();
      delays[n] = i-start;
      start = i;
    }
    break;
  case ABCAST:
    for(n = 0; n < samples; n++) {
      for(i = 0; i < granularity; i++) {
	if (abcast_l(format, target, REPLY, "%*C", buffer, msg_size, 0,
		     nreplies, "") != nreplies) {
	  panic("Unexpected abcast rval");
	}
      }
      i = elapsed_time();
      delays[n] = i-start;
      start = i;
    }
    break;
  case CBCAST:
    for(n = 0; n < samples; n++) {
      for(i = 0; i < granularity; i++) {
	if (cbcast_l(format, target, REPLY, "%*C", buffer, msg_size, 0,
		     nreplies, "") != nreplies) {
	  panic("Unexpected cbcast rval");
	}
      }
      i = elapsed_time();
      delays[n] = i-start;
      start = i;
    }
    break;
  }
  if (nreplies == 0) {
	  /* synchronise */
    int rcount = (nonexclude ? groupsize : groupsize-1);
    if (cbcast_l(format, target, STREAM_SYNC, "%*C", buffer, msg_size, 0,
		 rcount, "%-C", &replyptr, &replysize) != rcount) {
      panic("Unexpected cbcast rval");
    }
    finishup = (elapsed_time()-start)/1000;
#define TWOD(a)		(a/10), (a%10)
    printf("finishing took %2d.%d\n", TWOD(finishup));
  }
  getrusage(RUSAGE_SELF, &rfinish);
  
  utime = (rfinish.ru_utime.tv_sec - rstart.ru_utime.tv_sec)*1000000 +
    (rfinish.ru_utime.tv_usec-rstart.ru_utime.tv_usec);
  stime = (rfinish.ru_stime.tv_sec - rstart.ru_stime.tv_sec)*1000000 +
    (rfinish.ru_stime.tv_usec-rstart.ru_stime.tv_usec);
  free(replyptr);
  free(replysize);
  /* reduce samples to ms */
  for (n = 0; n < samples; n++)
    delays[n] /= 1000;
  /* dump the samples, 15 per line */
  for (n = 0; n < samples; n++) {
    if (n % 15 == 0) {
      printf("***");
    }
    printf(" %d", delays[n]);
    if (n % 15 == 14) {
      printf("\n");
    }
  }
  printf("\n");
  qsort(delays, samples, sizeof(int), int_compare);
  sum = 0; sumsq = 0;

  /* We omit the top and bottom discps% */
  if (nreplies != 0) {
    /* print rpc-like stats  -once w discard and once without */
    discard = samples*discps/100; i = 0;
    for(n = discard; n < samples-discard; n++)
      {
	sum += delays[n];
	sumsq += delays[n]*delays[n];
	i++;
      }
    min = delays[discard];
    max = delays[samples-1-discard];
    mean = (sum/i);
    if(sum)
      sd = (int)(sqrt((double) ((sumsq/i - mean*mean))));
    else 
      sd = 0;
    printf("g:%d s:%d r:%d p:%d d:%d%%\t",
	   groupsize, senders, nreplies, msg_size, discps);
    printf(" %4.2f ms  (%4.2fu; %4.2fs) [%2d, %4.2f+/-%2d, %2d] BCAST: %d\n",
	   mean/granularity, (utime/1000.0)/msg_count, (stime/1000.0)/msg_count,
	   min, mean, sd, max, bcast_type);
    i = 0; sum = 0; sumsq = 0;
    for(n = 0; n < samples; n++)
      {
	sum += delays[n];
	sumsq += delays[n]*delays[n];
	i++;
      }
    min = delays[0];
    max = delays[samples-1];
    mean = (sum/i);
    if(sum)
      sd = (int)(sqrt((double) ((sumsq/i - mean*mean))));
    else 
      sd = 0;
    printf("g:%d s:%d r:%d p:%d d:%d%%\t",
	   groupsize, senders, nreplies, msg_size, 0.0);
    printf(" %4.2f ms  (%4.2fu; %4.2fs) [%2d, %4.2f+/-%2d, %2d] BCAST: %d\n",
	   mean/granularity,  (utime/1000.0)/msg_count, (stime/1000.0)/msg_count,
	   min, mean, sd, max, bcast_type);
  } else {
    /* print stream-like stats */
    for (n = 0; n < samples; n++)
      sum += delays[n];
    sum += finishup;
    i = msg_size*msg_count/1000;
    printf("g:%d s:%d r:%d p:%d\t",
	   groupsize, senders, nreplies, msg_size);
    printf("%4.2f messages/sec (%4.2f ms (%4.2fu; %4.2fs)), %4.2f kb/sec) BCAST: %d\n",
	   msg_count*1000/((float)sum), ((float)sum)/msg_count,
	   (utime/1000.0)/msg_count, (stime/1000.0)/msg_count,
	   1000.0*((float)i)/((float)sum), bcast_type);
  }
  /* dump the samples, 15 per line  (they will be sorted this time) */
  for (n = 0; n < samples; n++) {
    if (n % 15 == 0) {
      printf("***");
    }
    printf(" %d", delays[n]);
    if (n % 15 == 14) {
      printf("\n");
    }
  }
  printf("\n");

  free(delays);
  free(buffer);
  cbcast (gid, DONE, "%*d", 0, 0, 0);
  test_done(0);
}
-------------------------------------------------------------------------