Path: utzoo!utgpu!watmath!att!dptg!rutgers!apple!brutus.cs.uiuc.edu!wuarchive!wugate!chris
From: chris@wugate.wustl.edu (Chris Myers)
Newsgroups: news.software.nntp
Subject: Useful NNTP Patch
Keywords: nntpxmit speedup
Message-ID: <237@wugate.wustl.edu>
Date: 30 Aug 89 22:25:30 GMT
Organization: Washington University (St. Louis)
Lines: 280

Here is a set of patches that I made to NNTP 1.5 and BNews 2.11 that helps
reduce the CPU time and I/O for transmitting news via NNTP.  I have been using
these patches on wuarchive.wustl.edu for several days now without any
difficulties.

If you use CNews, discard the patch named 'news-2.11-patch' and whenever I
mention a 'Z' flag below substitute 'n' instead.

The original nntpxmit program would open each message, scan the headers and
extract the message ID before sending the "IHAVE" command.  This actually uses
a fair bit of CPU time and a causes a LOT of I/O.  Since rnews already knows
the message ID it's much more efficient to have rnews write the message ID (and
the message path) into the batch file and just have NNTP read it back out later.

This patch won't do a lot for lightly loaded NNTP servers, but will help a
well-connected site a BUNCH.

What this set of patches does is:

a) Modify rnews/inews to add a 'Z' flag in the sys file (used instead of the
'F' flag).  The 'Z' option causes rnews to create batch file entries of the
form: "<path> <messageID>" rather than just "<path>".

b) Modify nntpxmit to check for a batch file which has both the message path
and message ID in the batch file.  If it finds both, it will not scan the
message for the ID.  If it does NOT find both (i.e. the old batch file format)
it will go ahead and scan for the message ID.  Thus the new nntpxmit is
compatible with the old one and can operate with either type of batch file (or
a batch file with BOTH types of entries for that matter).

Here is a shar file containing two patches.  The first, 'news-2.11-patch',
should be applied to ifuncs.c in ./news-2.11/src and the second,
'nntp-1.5-patch' is applied to nntpxmit.c in ./nntp-1.5/xmit

Chris Myers
Washington University in St. Louis

-------------------------------- CUT HERE -----------------------------------
#!/bin/sh
# to extract, remove the header and type "sh filename"
if `test ! -s ./news-2.11-patch`
then
echo "writing ./news-2.11-patch"
cat > ./news-2.11-patch << '\End\Of\Shar\'
*** ifuncs.c	Tue Aug 29 19:13:27 1989
--- ../ifuncs.c	Tue Aug 29 19:32:37 1989
***************
*** 281,287 ****
  	int useexist = (index(sp->s_flags, 'U') != NULL);
  /* I:	append messageid to file. implies F flag */
  	int appmsgid = maynotify && (index(sp->s_flags, 'I') != NULL);
! 
  	/* allow specification based on size */
  	if ((size_ptr = strpbrk(sp->s_flags, "<>")) != NULL) {
  		struct stat stbuf;
--- 281,288 ----
  	int useexist = (index(sp->s_flags, 'U') != NULL);
  /* I:	append messageid to file. implies F flag */
  	int appmsgid = maynotify && (index(sp->s_flags, 'I') != NULL);
! /* Z:   append name AND messageid to file. implies F flag */
! 	int appboth = (index(sp->s_flags, 'Z') != NULL);
  	/* allow specification based on size */
  	if ((size_ptr = strpbrk(sp->s_flags, "<>")) != NULL) {
  		struct stat stbuf;
***************
*** 362,368 ****
  			sp->s_name, oldid, hh.ident);
  	}
  
! 	if (appfile || appmsgid) {
  		if (firstbufname[0] == '\0') {
  			extern char histline[];
  			localize("junk");
--- 363,369 ----
  			sp->s_name, oldid, hh.ident);
  	}
  
! 	if (appfile || appmsgid || appboth) {
  		if (firstbufname[0] == '\0') {
  			extern char histline[];
  			localize("junk");
***************
*** 392,399 ****
  		ofp = fopen(sp->s_xmit, "a");
  		if (ofp == NULL)
  			xerror("Cannot append to %s", sp->s_xmit);
! 		fprintf(ofp, "%s", appmsgid ? hh.ident :
  			firstbufname);
  #ifdef MULTICAST
  		while (--mc >= 0)
  			fprintf(ofp, " %s", *sysnames++);
--- 393,402 ----
  		ofp = fopen(sp->s_xmit, "a");
  		if (ofp == NULL)
  			xerror("Cannot append to %s", sp->s_xmit);
! 		if (!appboth) fprintf(ofp, "%s", appmsgid ? hh.ident :
  			firstbufname);
+ 		else fprintf(ofp, "%s %s", firstbufname, hh.ident);
+ 
  #ifdef MULTICAST
  		while (--mc >= 0)
  			fprintf(ofp, " %s", *sysnames++);
\End\Of\Shar\
else
  echo "will not over write ./news-2.11-patch"
fi
if [ `wc -c ./news-2.11-patch | awk '{printf $1}'` -ne 1837 ]
then
echo `wc -c ./news-2.11-patch | awk '{print "Got " $1 ", Expected " 1837}'`
fi
if `test ! -s ./nntp-1.5-patch`
then
echo "writing ./nntp-1.5-patch"
cat > ./nntp-1.5-patch << '\End\Of\Shar\'
*** nntpxmit.c.old	Tue Aug 29 19:56:36 1989
--- nntpxmit.c	Tue Aug 29 19:56:40 1989
***************
*** 368,373 ****
--- 368,374 ----
  #else
  	char	*mode = "r";
  #endif	FTRUNCATE
+ 	char	mesgid[255];
  
  	if ((Qfp = fopen(file, mode)) == (FILE *)NULL) {
  		char	buf[BUFSIZ];
***************
*** 408,415 ****
  		*/
  		catchsig(interrupted);
  
! 		while((fp = getfp(Qfp, Article, sizeof(Article))) != (FILE *)NULL) {
! 			if (!sendarticle(host, fp)) {
  				(void) fclose(fp);
  				requeue(Article);
  				Article[0] = '\0';
--- 409,416 ----
  		*/
  		catchsig(interrupted);
  
! 		while((fp = getfp(Qfp, Article, sizeof(Article), mesgid)) != (FILE *)NULL) {
! 			if (!sendarticle(host, fp, mesgid)) {
  				(void) fclose(fp);
  				requeue(Article);
  				Article[0] = '\0';
***************
*** 449,463 ****
  **	Watch all network I/O for errors, return FALSE if
  **		the connection fails and we have to cleanup.
  */
! sendarticle(host, fp)
  char	*host;
  FILE	*fp;
  {
  	register int	code;
  	char	buf[BUFSIZ];
  	char	*e_xfer = "%s xfer: %s";
  
! 	switch(code = ihave(fp)) {
  	case CONT_XFER:
  		/*
  		** They want it. Give it to 'em.
--- 450,465 ----
  **	Watch all network I/O for errors, return FALSE if
  **		the connection fails and we have to cleanup.
  */
! sendarticle(host, fp, mesgid)
  char	*host;
  FILE	*fp;
+ char	*mesgid;
  {
  	register int	code;
  	char	buf[BUFSIZ];
  	char	*e_xfer = "%s xfer: %s";
  
! 	switch(code = ihave(fp, mesgid)) {
  	case CONT_XFER:
  		/*
  		** They want it. Give it to 'em.
***************
*** 753,766 ****
  ** Read the header of a netnews article, snatch the message-id therefrom,
  ** and ask the remote if they have that one already.
  */
! ihave(fp)
  FILE	*fp;
  {
  	register int	code;
  	register char	*id;
  	char	buf[BUFSIZ];
  
! 	if ((id = getmsgid(fp)) == (char *)NULL || *id == '\0') {
  		/*
  		** something botched locally with the article
  		** so we don't send it, but we don't break off
--- 755,770 ----
  ** Read the header of a netnews article, snatch the message-id therefrom,
  ** and ask the remote if they have that one already.
  */
! ihave(fp, mesgid)
  FILE	*fp;
+ char	*mesgid;
+ 
  {
  	register int	code;
  	register char	*id;
  	char	buf[BUFSIZ];
  
! 	if ((strlen(mesgid) == 0) && ((id = getmsgid(fp)) == (char *)NULL || *id == '\0')) {
  		/*
  		** something botched locally with the article
  		** so we don't send it, but we don't break off
***************
*** 771,776 ****
--- 775,782 ----
  		return(ERR_GOTIT);
  	}
  
+         if (strlen(mesgid) > 0) id = mesgid;
+ 
  	if (!msgid_ok(id)) {
  		sprintf(buf, "%s: message-id syntax error: %s", Article, id);
  		log(L_DEBUG, buf);
***************
*** 801,814 ****
  ** Returns a valid FILE pointer or NULL if end of file.
  */
  FILE *
! getfp(fp, filename, fnlen)
  register FILE	*fp;
  char	*filename;
  register int	fnlen;
  {
  	register FILE	*newfp = (FILE *)NULL;
  	register char	*cp;
  	char	*mode = "r";
  
  	while(newfp == (FILE *)NULL) {
  		if (fgets(filename, fnlen, fp) == (char *)NULL)
--- 807,822 ----
  ** Returns a valid FILE pointer or NULL if end of file.
  */
  FILE *
! getfp(fp, filename, fnlen, mesgid)
  register FILE	*fp;
  char	*filename;
  register int	fnlen;
+ char	*mesgid;
  {
  	register FILE	*newfp = (FILE *)NULL;
  	register char	*cp;
  	char	*mode = "r";
+ 	char	buffer[255];
  
  	while(newfp == (FILE *)NULL) {
  		if (fgets(filename, fnlen, fp) == (char *)NULL)
***************
*** 822,827 ****
--- 830,840 ----
  
  		if (filename[0] == '\0')
  			continue;
+ 
+ 		if (index(filename, ' ') != NULL) {
+ 			sscanf(filename, "%s %s", buffer, mesgid);
+ 			strcpy(filename, buffer);
+ 		} else strcpy(mesgid, "");
  
  		if ((newfp = fopen(filename, mode)) == (FILE *)NULL) {
  			/*
\End\Of\Shar\
else
  echo "will not over write ./nntp-1.5-patch"
fi
if [ `wc -c ./nntp-1.5-patch | awk '{printf $1}'` -ne 3781 ]
then
echo `wc -c ./nntp-1.5-patch | awk '{print "Got " $1 ", Expected " 3781}'`
fi
echo "Finished archive 1 of 1"
exit